{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 参数更新"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `SGD`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SGD:\n",
    "    def __init__(self, lr=0.01):\n",
    "        self.lr = lr\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        for key in params.keys():\n",
    "            params[key] -= self.lr * grads[key]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 如果函数的形状非均向，比如呈延伸状，`SGD` 的搜索路径会非常低效\n",
    "- 原因是，`SGD` 梯度的反向并没有指向最小值的方向。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `Momentum`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Momentum:\n",
    "    def __init__(self, lr=0.01, momentum=0.9):\n",
    "        self.lr = lr\n",
    "        self.momentum = momentum\n",
    "        self.v = None\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        if self.v is None:\n",
    "            self.v = {}\n",
    "            for key, val in params.items():\n",
    "                self.v[key] = np.zeros_like(val)\n",
    "        for key in params.keys():\n",
    "            self.v[key] = self.momentum * self.v[key] - self.lr * grads[key]\n",
    "            params[key] += self.v[key]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `AdaGrad`\n",
    "学习率衰减`(learning rate decay)`，随着学习的进行，学习率逐渐衰减。\n",
    "- `AdaGrad`，学习率除以以前的所有梯度值的平方和，元素中变动较大的元素的学习率将减小"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AdaGrad:\n",
    "    def __init__(self, lr=0.01):\n",
    "        self.lr = lr\n",
    "        self.h = None\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        if self.h is None:\n",
    "            self.h = {}\n",
    "            for key, val in params.items():\n",
    "                self.h[key] = np.zeros_like(val)\n",
    "        for key in params.keys():\n",
    "            self.h[key] += grads[key] * grads[key]\n",
    "            params[key] -= self.lr * grads[key] / (np.sqrt(self.h[key]) + 1e-7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `Adam`\n",
    "融合 `Momentum` 和 `AdaGrad` 两者"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Adam:\n",
    "    def __init__(self, lr=0.001, beta1=0.9, beta2=0.999):\n",
    "        self.lr = lr\n",
    "        self.beta1 = beta1\n",
    "        self.beta2 = beta2\n",
    "        self.iter = 0\n",
    "        self.m = None\n",
    "        self.v = None\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        if self.m is None:\n",
    "            self.m, self.v = {}, {}\n",
    "            for key, val in params.items():\n",
    "                self.m[key] = np.zeros_like(val)\n",
    "                self.v[key] = np.zeros_like(val)\n",
    "\n",
    "        self.iter += 1\n",
    "        lr_t = self.lr * np.sqrt(1.0 - self.beta2**self.iter) / (\n",
    "            1.0 - self.beta1**self.iter)\n",
    "\n",
    "        for key in params.keys():\n",
    "            self.m[key] += (1 - self.beta1) * (grads[key] - self.m[key])\n",
    "            self.v[key] += (1 - self.beta2) * (grads[key]**2 - self.v[key])\n",
    "            params[key] -= lr_t * self.m[key] / (np.sqrt(self.v[key]) + 1e-7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 优化算法比较"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAERCAYAAAB7FtAjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9d5wcxZn//67unjyzs7uzebVarXLOBCMyRkQbcICfjRN3znd8HTgfPhsbh7MB24A5c9hgbBNtDOZsMGAyJggJISEhJJS1QZvT5Nihfn/0bGJXQgKBLNxvvVrT211dXdPzqXqeqq4gpJQ4ODg4ODgohzsBDg4ODg7/GDgGwcHBwcEBcAyCg4ODg0MRxyA4ODg4OACOQXBwcHBwKOIYBAcHBwcHwDEIRyxCiMuEEG1CiD1CiHOLxy4UQrQLIXYKIU4qHvueECIlhIgJIZ4RQsw/vCl3eC8jhLhNCJEUNpVCCCmE+N7hTpfDgeEYhCMQIcRS4JPALOBDwG+EEA3AjcApwIeB24QQavGSG4FK4GngESGE591PtcM/EUFgMjDv3bqhEGKxEOL8d+t+71Ucg3BkMg/ok1JmpZQbge8A5wPPSSl3Sik3ARlgztAFUkpdSvlDwMA2Gg4O7xS7sTU6r7j/brAYOw84vA0cg3Bk8hxwrBDil0KIOinlLUAT0DwqzH8D8QmufRWY/S6k0eGfl82MGITNAEKIbxSbOLcLIc4qHpNCiDuEEM1CiJ8JIfqFEKcKITQhxM+FEJ1CiG1CiGOL4W8TQlwhhHhRCDEohPiP4vEW4AbgIiFEtxDiu6PCf2YoUcX7TSmGeVQIsUEI8SshRJ8QYg4OaIc7AQ4Hj5SyVQhxHHAtsEMI8W9AGGgfFeYPAEKIN16ewq7SOzi8U2zFNgaNwFrg/UApsAioBZ4WQiwqhr0FmAvEgNuA47GbQsuBKcAJwM3FawG+AJxcPP8k8DMp5ZRiwX+ylPIzB5C+6mI8LwMvABXAUcV0/1PjGIQjFCnla8BKIcQHgXuAZ4E+IUQl8BrgBT4ywaUBbKPg4PBOsQu7YHcDWeAl7CbOKBAVQryEXdADrMbW4+riMQXbgJwKtBXD+IQQQ2XV7VLK3UKIPUDJgSZIjPWM2otOVaZ43/fjtJYAzkM4IhFC/LcQ4tMAUsoHgWeKW5OUsk9KWQOsZ2KDvwDY8q4l1uGfERO7sO4fdUy+YV8CSCnNUdcMIYAvSilrilpuGnV+d/G6g52Vs/4N6Zto/58exyAcmbQBlwghfEKIKuxC/gHgDCHEnOKxxaMvKLbLfhM7s/393U6wwz8d2xlpgjkGuFgIUSqEmF38+4X9XPsk8EkhhFsIsRDYxkhZtS9D0I/dRIUQoqJ4LAE0FPcvfUvf4p8Mp8noyOS3wHJsbykPfF9KuV0I8WXgMWCAsb07/h27zXQNcIaUUn+X0+vwz8dWYC8wCbuATwObgBzwL1LKngnebw1xC3YPuWYgCXxcSmnuJzzYuv+sEKIH6MJ2iH4N3CuEWIHtMDm8CcJZD8HBwcHBAZwmIwcHBweHIo5BcHBwcHAA3kWDIISoFkI8X9x3CSH+KoRYJYT4l3crDQ4O7wSOth3eK7wrBkEIUQbcjt0HHuw3/uullCuAjwghQu9GOhwcDjWOth3eS7xbvYxM4CJG3vSfDHyzuP8cdo+ZZ0ZfIIT4PPB5gEAgsGz2bGe2BYd3hvXr1/dLKSvf4uWOth3+YTlYbb8rBkFKmYAx0ygEgI7i/iD2UPI3XnMLdvczli9fLtetW/fOJ9ThnxIhROtbvdbRtsM/Mger7cP1UjkF+Ir7wcOYDgeHQ42jbYcjlsMl1vXYc52APWlVy2FKh4PDocbRtsMRy+EaqXw79kItJ2DPdPjSYUqHg8OhxtG2wxHLu1pDkFKeXPxsBU4HVgHvHzXBlYPDEYmjbYf3AodtLiMpZSdw7+G6v4PDO4WjbYcjFeeFl4ODg4MD4BgEBwcHB4cijkFwcHBwcAAcg+Dg4ODgUMQxCA4ODg4OgGMQHBwcHByKOAbBwcHBwQFwDIKDg4ODQxHHIDg4ODg4AI5BcHBwcHAo4hgEBwcHBwfAMQgODg4ODkUcg+Dg4ODgADgGwcHBwcGhiGMQHBwcHByAw2QQhBCaEKJNCPH34rbgcKTDweFQ42jb4UjmcC2QsxD4g5Ty8sN0fweHdwpH2w5HLIeryehY4FwhxFohxG+EEIdt5TYHh0OMo22HI5bDZRBexl5v9mjABZz9xgBCiM8LIdYJIdb19fW96wl0cHiLONp2OGI5XAZhk5Syq7i/DpjxxgBSyluklMullMsrKyvf3dQ5OLx1HG07HLEcLoNwpxBikRBCBc4HXj1M6XBwONQ42nY4Yjlc7Zs/AH4PCOBBKeWThykdDg6HGkfbDkcsh8UgSCk3Y/fGcHB4T+Fo2+FIxhmY5uDg4OAAOAbBwcHBwaGIYxAcHBwcHADHIDg4ODg4FHEMgoODg4MDcPi6nTq8BQp5Hcuyhv/e2tpGrlCYIKQA5Lijsxoa8Hs9w397fZ5xYRwc3m2klORzIzpOprPs6uyYIOTEug543cxsmDz8t6IquN2udyCl730cg3AIyGbyrN+5g43t2+hMd5Ew4uSVNJaWAVcB1aWjuXQ0zUDTTDTVxKWZaIqJqlhoioWqWCjC3leQaIoFSFQhUbBAgCZsYyAAkCgq4DvwdG7vA4kYzlKmVJDSPmZKBTl8TGBYCqZUMKSCadmbbqoYpoppqva+rmEYGqbuQupuhO7FZfkJEKbSV8GsSBPHz15IRWXpoXzcDu8iPT0DvLDjNXb2t9KX6ydDHF3NILUcwlVAc+momoHmMnCptrZV1cSljtK2sDdNsRBCogoLAcVPiRD2PtjaFkjEQegaYP1eilfaGFIBCRYKphS2xi0FSwpb01LBshQMS8GwVAxTwSjq2zBUdN2FaWhYugupe1AMHx4rQIlWyqRgHcsmz2HhtGn43mNO1T+vQbj7bvj2t6GtDSZPhh/9CC6+mFQqy0NrX2RT32YGrB50dxLNl8HtzeH1FPC6dbyajkfT8agGHsXApZiofkloJsx6k9tKCRZiuCCGos9TVLIo/jd0RgBCDH2+va8sxOgsAwomUoCUApcwR4yFBKkKEBIQw9eMpEWiHEBa4sBDUdAHVfKWRt7UKJgaWd1FXneRK7jJ5zzoWR9KLkRQltMUmsZ5i06isaHq7X3Zf2b2oe3dbZ38deNzNKebSSuDSE8Sly+Hx5vH5y7gGdK2auBRDdyKgVsxoQZqa6D2TW5rFTUt5YjKZNGrF0MHihocks+h0ratxxFtC8ziXxaymKmkYgcZHUqIIW3bhkkpfr4Z24AtewUFS6VQ1HbOcJE3bF3n8i7yWS9W1o/LKKFcqWFpzULOXHY0wcBBWrt3ESHl+CrYPxrLly+X69ate9vxbNq5m/s3Ps6kZx/hU7c+iidvDJ/TvQrP/GAm7eeVo4qJn4klbQ/aKnocSFvQKhJFWCgH9EZGw57zzIWFiiW14qcoeuQCc3gDQ4I5vMnhfQtZzIB2gQ7jK9OjMx2AgkARElUIVAEqAlXB3hegCVCFRBPYtRVhfy8VC0WYCAwEBqADEzVVjcewBFbxmQ0VBoqQxXTsW3tZUyNrusnobtJ5D+mcl0zKj5UuodSq44T6Yzjv+BMQb7ckAYQQ66WUy992RG+BQ6Ft0zS5//m/s7rrZea8+Cyf/vUTePLm8Hndq/DsD2bQcX75vuMoak4WtS0kCGGhComqHGgZ4QZcSDQkGpZU7fxCsaYpR+5jWGBi69uywEQOa9ySAquo5rHaFmP2YMSgKMLW95CWhzU+Stf2vizuW8PaFsJAwWRE1yPPbl9YFlhypPYxpG21mGf25TCZUpAzXWQMN2ndQzrnIZ3xkU8HULJl1KlNfGzZmcyaOnniCA6Sg9X2e8ogWJbF7x5/mJcH12IGe/CHk5T4M5R4s4RceXyqDsCHT15PsHN8gZao9fD7J49BExaaaiIEND3Yx9Lr2gh0FUjXunnl6020nteEhRdTuilYLvKWQsYUZAxJypRkTIluqehSxZAquqViSAVDqsO+kYKCJjRUoaGgoqBiv+MXMCpjWpbtbdkCLHphEiwpi8ZgyDOzvfk3MuR9iWKtQxGgCGF7Q6M2IeTwJ0JiZ0l7szAxMTAsA3M4s0j7OQkLTZi4FBOXMNGEiVeVBFWFgCbwqxKvInErBi5RQBF5pMwiyU74GxqmXY23hp6TUryPYo0LW7BUUrqHZMFLIuMnmQxgJiNMUmbxhZM/RG1F5E01A0eGQejo6uVXz99PJztxlQwSDKUJ+zOE3DmCWh5X8fnsS9vJWg+/f/JoNMVEUyfO8wo+EH5M6cGQLgqWRtZSyBiQNiUpwyRvKbauR+u72AQzpD9VqLa20VCEipAKtraVYW3bOhbDmh7R9mhHp7hNkNYhpSujta3Y9dlhXStFB0QZqQEgLMACYWHJoralgS71Yh6yw2rCRBPWGF27FJOACkFNxa+CT5V4FAu3oqOJAoIcUqaRGOPSa0ns5lapYMFwM5lLMSc0HhnDRVL3ksx7iaf9ZOJhXJkqjq04lk+edibKgXmfB63tI7LJyDRNfv3Yg6xLrcJV2kd5OEGZP03YncU9zeR900bCFiyVgqlSMFR0XcGlWAS6JvZuQ915Ap4p5CwPMUOh4S87OfYHzbhzdiEY7Cxw9Ld3sybWyLOnN3D6k3u4+JYXqOiNM1BVyn1fOJPnT1tGwZBkdUnGsEgVTNIFY7j90rIElqWMqjSPoAqBW9FwKSpuRUEVCppQsP8JuyAf8t4QY2OQ482BFMWMNRIEhJ3h7E85XOwb0sKUFro00S2Lgmmgy/GFMEgUpejhKxaKIvFqgqBLI+BW8bkEXk0QU+3zCBOTAgUrT8bMDHt+ChYexW5yC2mSiEejVBOENInPreMSWSCBYcWRjE1HQdfImyqWsJu6qv0JGkODUD0U4kUeHbyNRI+XaM5PNBkkNVhOpT6Ty07/NJXl/7jvNLr6B7juydsZ9OwiVB6lLJSizJsmpOWYvgSmF8NJCTlLQzdVMrobFYlbNQhMYAwAgl15vO7pZEyNwYJKVLcYKBikDEHO0shbGrLY6VBBwa/6cCseVNwg1eI7JEHOkGR1i3TBIlUwyBsSU9qatixh1wQn0LZLKLhVDbeiohV1rYqirifQ9hsZU5kUAiltwY+4J7ZTZDeBWtjFvu3ODGm7YFnolkneNIZ1OEKxyUhYw/rWFAu/WyPkVvG7FLyaQlIDlypRFBOJgS7z5KwcOSs/HI8mLDyKjlcxKHOrlLtUSl0Q0EwCSgGVNJaMY8r0mBRYFuQNF7pUkUBAy1PmyeAu7YP6oVBPcuvOq4gXfEQzAaLxEozBao4rO5HPrDznbdeYj4gawpSZ0+VpPzqXUFUPleE4EW8KnzpihQ1LIW24KRgawgKvauH35FGUkaqfSwngVqqxCHP28XcS7EyOu09PdYh/ufczhDQ/IS3E1eddS6Q7Oi5cRyTMNR9ZydW/+wv+gj58POt2cc0XP86LpxyPT3GhoaBKBWmCZUgM3aJQMCkUTLIFg1xeJ5MzkJZkqMybKDO8GYooZigxcrX9LsAu9M23+BvLISujgNet4fNqeN0aHo+G26XicimoLgWhgqVIDCwK0iBt6iT1HLFCDmvMve3mhxKPSqXfQ5lPI+RW8LgkiqJjkCdlpogVEsMZViDxKwb1Pg81XhcRN7ZHLFIYVj85s38kdgn5gpeMoWECqmbh1wp4R2nFlILBfIC+VIiB/giR9Dx+8rGvHrYaQs3UBvmB/zmTSMUAlYEkZZ7MmKa0nKmRMdwYhoqGxK/peNz5Me3cPrUSValAlwEunHEjijn+9zYVwfnP/D/K3CUE1AAaXixLI6cLknmLaE6nL5MnmbcwrbGFuiIEZW4fIZcHv+rCIzRUqaBIgTTANC30gq3tXMEglzfI5nXyBdPW9T4K+TdDMKJtRmu7WDu2ivo+WIZqAiigKMLWtceFz63h8ai43CqapqBoAqGCKSx0LHKWTsooEC9kSRv6mDgFEpcmqfC5ifjclHpV/G6BW7OQokBeZkkYCVLGSM1YFSblLqj3eqj2qoRdFn4liyBO3uzBkJnhsKapksl7yFsKUgW3ahB05cdoJWO6GMgF6YuVkOqt4aTylXz69LPfgzUEdw8nLl4LQMpwM5gNYOoaPiGoDJpoWpSwOwdu8KkRQq5GJBXEDS9dOcmedIbWTHzYz2z7l+O49GdP48mN/KgFj5t7PnMhA70z2JKxLffNExgDgLrBBFf++e9jjAGAr6Dzudsf5J6yRZzz2nq+/tQj1MajdIfL+PXZF7BuxUmU+wIEAi48IdtTUqRAkYAJlmlhGRLTsDB0E0M3MQ0LvWBiGCaGXjxuWJim/WlZEtOSWJYczihDKEJBCIGqCBRVoCoKmqagqgoul4rqUtBcKi5NRXOrw8fUYhih2YW9VMASYEiLgmWSMw0S6TyxbI5YNkvmDc9hiIg3SEWJn3DQSyDgxuvTUFxgKhZZqdOfybCrP8FALjPqqgAhVxlTwgHqQl7KfBo+l4ml5OjIx1gb7SVnAQSBIBH3PGYEAzT4NMpdOm5/nKzRTlJvH65ZKASIJX3EC4DLJODJMzvShVLRBWw+EAW+Y/gCUY6ftQVLQrTgpzNZitBVSlxQWpLBq2bwqgbCo1LiasCn1ZOXYQbyGm1Zg52pJFF9pJC5aAJjAKBYkuaOBl7TdWx3wb6mwuunLhBmksfPDK8L1VKwCha5nEkqXSCWzDKQyJLKF0iTGRdvwO2m1O/l3E3r+eSD9xEZ6MdSFBTLIlpRyaMf+RSvHHsy0pCYpoWhW5hDWi7YGtdHa9u0bP2bJtICs6jr0QW/QKAoYkTbiijq2i7INZeK5lLQtGLh7lLRNPuYog1tAoq6toRElxZ50ySjF4hHcwxm08QyOXRrfC1ZESoNwSDlJX5CQTcBvwu3RwUNdGGSMPN0xlN0pOLkzCFnRAXKqAs0MLnER3XATYlPoKkGBdJsSgzQkxvAQgMiKFQwJVDCVL+fWq+gxJdFBPpJ6M3kzNhwWqReTl9KIYdEdRmUe9M01EehvpWsuf5g5Xhk1BCqZ0XkF391DgvqQxjqHvJWHLAL/wrvPEKuaUT1ALvTOlsSXbSku4a9gCpPOdOCk5jkq0ZaXvrSFrujSWr//BCfvfsB6gdiZN0uvv/ZC9l+zkqmlpQzKRDGZal86pwPEe7pGZeejnAZtfHohKP6pBA8ffmVnHj9Vbjy+eHjeZebx2e9j6OaN1GVjtITKOOmxWcB8OWNf6O6eOzmpWfz4rzj8Hvd+DwuvB4Nr9uFx130yjX7U1MVNNUuuBXFLuxhqD1V2NVnCZYlsSwL05IYpoVhmOimSaGYEfMFg3zB9uhyeZ1MXieTLZArjG8HHUIICAd9lJf4qSgNUFbqJ1jixRdw4/JqKG6BjsVAJktvMkVXPElXPMlgZux7g5DHQ1NFGZMjpVSU+fAHXOCGvnya5sQgu+MDdGVGanIhl4d5kSpml5VSW+LG7zbpK/SzJ9VBW6YLs9jEFXGHmRuezOygn1qPgWF10J/fQqzQAkgUNEpds9jabtCdNbl+5Z2Hr4Ywq1x+47fnM7NeJapvQ2IiUCh1TyXinYOi1NGV1diWSrEl3kZMt5+HJlQaA7VMDdRT4a4gnVfoSBS49AOfprJ3YNx9JHD/9y6n67wPIAuQTun0RzO0DsRoGYiSyo9taqoI+KkNh6gpCVFVEiTi96FJBVmwKGQNsukCTU8+wql/uZ2ymF1Lmyg/WNiefnegjFuXn8vq+SvweV22tt0anuLmdmm4tWLBPaRtRaAoSrHwH6trANMqOkSmhWFa6IaJblgUdIOCbus6rxtk8wbZXGFY24Y5UVOojUtTKQ/7iYT9lIdtbQeCbjwBF5pHQ2qQMQz6Umm6E0m6Eym64gmy+tj8UhcOMSVSxqSKEsIlXtw+lYJi0J6Osyc+yO7EACl95JlPDpYyP1LF1NIAFQEFqeRoy3TRnO6gLx8tPl9BU7CeeaFqpgY1SrU0CX0X/bnNw0bCq5YhCo1s7kqRitdx/QXXvPdeKjctCMnv/d8CfGoFdf6jqfEvRVOmsG6wkzWDr7E90YKFxKu4mVPSxNzwVGaGGrEMH2u7u3iuq5mNfZ0ULLsJqTFUyoJILfPKq7ngymuIPPs8Dz32DK90dLOpo5udvQNYUnLexrVc/cA9Yyq7htdLxzU/o/aaq3BPMHimK1AGQG16fO3ija99Lc0FAhRdHxMGIFtRzfoP/Cub5x5PJp0nm86Ty+nksgXyeQM9b6DrBoZhF/KWYdkvmoe6rxZfHitFo6FqCi63hsul4vG48HhdeH0ufAEPfr+bQMhLMOQlVOKjJOwnGPbhDrhx+1zkTYtEKkc8lSWeyhJNZIkmMgzE0/TH0vRFUwzE0pjWKC9OQHV5iMm15UypK2fapAiT68rwhtz0JNO0DcZoHYyxp3+Q3X2D9KVG2lNrSoIsqK9hSUMd8+orUbyCHfF+tgz2sHmgm63R3mHPbVo4wvE1jRxf10htyEVzup2tiWa2xHfTX7AzSbU3wrGR+RxTPoNyV5Ke3Ea6Mi8zmN8BwGdmrj5sBqFpQVB+7/8WEfHMpta/nGrvYvoKAdYMbuelgc3DhUGVp5x54anMLmliir+e9kSBFzpbWdXVQnPSDuNWVL7w6m6+/rObJ2yg6S4t5+SvXDH8d3UoyLTKcpoqymksL6WxvJSqYIBMIk9rZ5Q97f20dA7S2jVI72AKgJXN64cdmInfFuwb0+PltS99mx2LTyaVzJFJ5WxtZwpkswUKOYN8XqdQMIZrx6Zpb/IN2j6+eyP/366/UZGLMeAr44GFH+TVmcfh8Q5p240/4MYf8BAMeQmGfITCPkpK/fiDHlx+N6pHJZPTiRe1HU1kiSUzDMYz9MfTDMTS9EZTJFK5Md/D41JpqCljck0ZUydFaKqPUFMdxtQke6PxYSO7p3+QPf2Dw8ZCFYKZ1RUsmlTL0oZaGqvLGDDSbI32sWWwm00D3bSn4sO/5fKqSRxfO4VjamoxlRTbk61sTTSzLdFM3tJRUJgbbuKY8gUsK6uiYLbQnV1PZ+ZlcuYgAa2aj079y3vPICxYMlM+u+ZhSlxTeWngNR7qfJ5N8Z0ATAtO4qjyeSwrm8PMUCNd6SR/3LWJB5q3sLf4cOeXV3NcTSNHVzewtLKeMo+PV9u7eez1nah33cl/3HUr53/hMjqnTGVhfQ0L6muYW1vFop3bqDrvA8iSEkQiQbYkzB/P/DR3hGew4vXVfGvNffjMkcJcd3vYcNmVHHX1txGH4LkOeVcWAgWJJRQUaRELRXj6+IvYvuhku6qsqSiKXfjbvYqEbRyGageGhWFY6LqBXjAp5HXyOX04I2bSedKpPNY+PCd/wENlTZjqmjDV9WXUNZTTMKWCxqlVVFaXIITAtCz6Y2m6+xN09sXp6ImztydKa1eUls5Bsnn7OamKYHpDJQtn1rF0ziSOmjuZUMBLNJNle08/W7t62dzZw6vtXbTHEgCEvR6Onz6F0+dM5+SZUxEKbB7o5uXedlZ3t7G2dy9ZQyeguVk5eQYfnbaAY6sn050f4JXoNtYNvs7G6HZ0aVDtjXBmzXGcVbsCTeRoT7/IzNIPHDaDsGDJDPnyy2vJWxqPdK3ise7V9OWjeBQXS8pms7x8LktKZ1PlLef5zmbu2/0aT7XvGv6+x1Q38L6aySyvamBeeTWGaeHzuCcsqKUQ3PniOubUVDGzuoKwz0s8meXl19t4ZWs7r+7oYE/7wHDzjN/rpqmunMm1ZTTUlLF8w/Ms+MmVKLmJe4gdCBLodYW5fdJpvDzlaPxBDz6fG6/PLsjdHhduj2bXFDTbkVEUBaHY7xJmbHiGU/7yv7j1/JjvKIG8y8v9Kz7B6vrl5LIFspkC6VSObGbiF+1CCMoiQSqrS6iuK6V2Ujn1kyNMbrK17Q/Yg85yeZ3ugSRd/XE6exO098Zo64rS0jVIZ298+HmF/B7mTath8axJLJ/XwNypNQgEe6NxtvX08XpXL68Vnc6hGtn0ynJOmTmVM+bNZF5tFQO5DOv7Oljbs5dV3S1si9prbjeFyjhv6jwunL6QSp+f7YkW1ke3sXZwMy3pTgSCJWWzOLfuBJaXzSFW2E3W7KchePx7zyAsX75c3vrE3dy8+37aMt1Ue8pZWfM+Tq5aTo3P7lrYkojy0w3P8kjrNoQQHF87hXMaZ3PqpOlU+gIAFAyD+zds4Y41G2geiOJSVVaG/Vz7lc/R+53vUvG976EU+4BJKen50v+j6tc38ZFPXsXtv7+SpxsXcd+F/8aC6bXMmVrD+/58G5U3XDuS0F/8Av7932HKFGhtfbcfE0QicMMNcPHFB32plJJspkAykSURyxCLpokNpBnsT9Lfl6CvO0Fvd4yu9iiZ9EhTWLDEx8w5dcxd1MDio5qYu7ABVVPHxG1Zkq7+ODta+9jW0sPmXV1s3tVFrmCgKoJlcxo4c8UcTjtmJt5RUw70JFKsbWln1e5WntvZzGAmS9Dj5kOL5/Evxy2jJhwCIGcarOlu49HW7TzSto1EIc/s0kouW3Ii7580HSEEGSPHmoFNPNH9EpviO/GpHi6avJIL6k/FpWqHzSAsW75MXnb/f/OnvU+RtwosLZvN+6uP4ZjIfLyqByklj7Ru57pXn2d3fIAyj4+zG2dz1uRZHF3dgFu1n3V7NM5vVq3jgU1b+etPvkt9fIL3X5EI9PeTzek8vmYbj67aysbtHVhS4ve6mD+9jnnTapjTVM2MyZXUVpSM7bVyCHUt/X7ELbccnFa//GX45S/3H8btht/+dky8pmGSSuZIJrLEoxni0TSDAykG+pIM9CXo7Y7T0xmjpzOGOcopqmsoZ9a8euYvaWTZsdOonTR+HEeuoNPcPsD21l5e39PD5l2d7G63m+xKQz5OWjadD5w0j/nTaoefpWlZbO/pZ/WeNl7Y1cLLrR0YlsWMqgifPnYp5y2ag6v4u/ZkkjzZvouHW7axursVRQg+2DSXyxafyKRgGICObC/P9Kzjie419BdiTA3U88XpH7tijvEAACAASURBVGFeeNp7cxzC1IUz5Zwbz6TOW8mnms7huIrFqGKkxfL+3a/xrTWPoQmFT81eyqdmLaU2UDImjs2dPVz2p0doHYyxsL6Gi5YtYOXcGYS8Hpg/H+rq4PHHAdjW0sNPb3+ar//icizNxf3fu5Ev//F6KnZvQ2ltHRlW+f3v29uTT8Jpp6H/+lYe8s9l4IabuWTtHaijXvFOPErgHUAIrv/ZfXzt6x9+R6KXUhKPptnb0k/Lrl527+hm+5YOWnb1YFmSUNjHKWcu4MMXH0dNfdk+4zEMk827u1m1cQ9Pv7yT9p4YZSV+vvTRFXzwpPnjus+ZlsXalnbu37CFR7fsQFMU/uP0E7j46EVjwuYMnb+2bOWXm9ewJzHI+U3zuOa4s/CoI/0nmlMd3Nn6MC8NbGZ+eBo/WXz4ehlVz22Qy391AcdXLObiKWcz2V8zfC5r6Hzthb/yaNsOZpVW8qX5x3JW46wx30VKyW9WreOGZ1YDcM78WXyudQdT//MyhD72Zb90u1nzte/y3XgZiVSOKXXlnHLUDFYsamLO1Bo0dR992+++G+Pyb6J2tB96DTc2Do+k3i933w2f/CTXH/cxvrbq928eZ0vLQSfFNEy6u2K07emjeWcPu7Z1sfW1dgb77fc202bVcNYFyzjjvKW43fvujxNLZlm7uZXnN+zm+Vf2kM3rLJlVz39+5jSmTqoYHz6T47HXd3DPuk1s7e5jRmWEaz9yNjOrx4bdm4px+7b13LV9A4oQ/GzFOZzdOHsk/dLk2d713NnyCH35KJ+dej4XNJx6cNq22+be/Q34DbAauOLNwpbMqpI/3Xq7zBl5+UZWd7XKKbdfJT/22O9ldzox7ryUUrYNROXSH90oT7nu1/K5Hc3SsqyxAb76VSm9XimzWfnq9g55/CU/lxf968+kJYQsXPEdO8wvf2mPk9m2beS6446T8uijpTRNaVRWydWTlsqVS78rv/6J/5WGqkkzEJAWSOl2S/mlL0npcg2PtZEw/u9DtDVe/tCEz+GdJJnIyOee3CJ//F/3ynOO+b48930/kM89sfmArrUsS768pVV+4Yf3yKM/ca383z8+v9/wewdj8vN3/VnOuvI6+Ye1GycMUzAN+fONz8vG26+SV770+IRhnu5eKz/w3FcksE4eJm2HZ1XJVX0Tf4f/eOEhOeX2q+TNm9dIwzQnDHPrCy/LWVdeJy+950HZFRul/0hkQm10Bsrkv1/9J7lxe/v4fDARd90lDY/3wLQnhP2pqmP/frPN75fyrrv2m4ahOBsvf+jA0nGIsCxL7m3pk/fftUr+28W/kiuXfld+/qM3ysH+5AFdn8rk5R8fe0We8aWb5En/+j9ye0vPfu/15NZd8vif3iyPvvom2R2f+B7tyZj80CN3yKl3XC1f7escdz5j5OSPttwqz3720oPW9oEK/ELAczARv0l8HwJuK+7/Fpixv/DVcyZJwzImfDiXr3pELvzD9TKjF/b5oG/6+xo568rrZEc0PnGAhx+2H8UTT8j/+p+/yjO+/EuZuv0u+9iLL9phdu+2//7FL+y/o1FbpFdcIaWUcvvSU2RM88v1L+6Q8qmn7LAPPCDl979vC7SvT8olS6TUtBHhXnGF/enxvOXC/x/FIIymrycuv/Sxm+THz/zZQV1nmpb89o0PyeM+fb3U9Yl/79FhL/r1H+QHb7pjv+G+9vyDcv7vr5vw3B//+Ee5qnPDITUIB6vtpgXTJ0ybYZpy2h3XyG+vfnS/3++062+Vl9z+p/GF+z4KYwtxwIZANjYeWOELdtg3FuoHGseQEZno+jcYtgMyCI2Nb/793gKWZckX/75Vrlz6XXnbTU8d1LW9g0l56udvlD+8Zf+/p5RS7uzpl7OuvE7evnr9PsMk8jk5666fyu+vfWLC83mzIJ/sfumgtX2g6yHMAZ4RQtwshFhxwNWPfXMycG9x/3Hg+DcGEEJ8XgixTgixLmcW2JsZ3/0ToMofJKnnebW/a583qwkHAXhi666hTDuWk04ClwueeIK6yhLiySw999yPDIfhqKPsMFOnwrRpw81KPPMMmCasXAlA5viTCRsZXrrhHvrvvA+paXDKKXDGGbZMH3kEduyAz38eNm604/jb3+zPb397f8/qgLh+xceZcvlDTLn8IQCmfPNhpnzzYa5/Ysfbjvtg6O9N8OC9a2nb00dD0/jq8f7Y1tLDazs7qY6EUPfVfFFkR28/zQNRakpC+wwTz+fYNNBNtT844fnNr2/morM+DNB4iHQNB6ntRDaFYY2fO0cRgipfkI39naT0/LjzQ9SEQ+zs7ae5/w3vDCZPPBeOJQTd/3Pz/r/B3XfbOn2z9wWNjXDnnba+W1rGN/tcfLF9/K67wO/ff1ymCZ/4BASDUFFhN8t+4hMwMDBe28X961d8fHw8brfdBHWIkVKyeUMrf7nnJQAmN1Ue8LWGafHoi1tJZfLUVYbf9D5PbN0FsF9tr+trJ2ca1PgnDtOS7uQv7c8ccBrHJOBAN+AkYDewE/jMwVz7hnh+Aywq7q8Evrm/8OWza+RFqy6Xz/W+Ms67ieWy8pQ/3yxn3PkT+fONz8tUYXyzkm6Y8ku//4ucdeV18pLb/yTXt05QXT7pJCmXLJGZbEF+5Zo/yY5AmXxpxlL5x8dekQOxtB3mi1+UMhiUMp+X8gtfsPcLds3E6uqSEuSdU1bKXb5q+XpkmvzF1Q/JZ//2qjTLyqQ1fbrtvTz4oO35KMqIh+V2v+1aweGqIaSSWbnhpd3yzpufkV+75FZ5xrIr5RnLrpQ//q/7ZHzoue2HfEGXz63fJS+79s/y6E9cK8+59Fdya3P3PsN3x5Py6keflfN/cIM84Wc3y+b+wXFhLMuSj7ftkCvuv0lOv/Ma+XxH87gwbelu+fUN1w5Vq7cdCl3Lt6DtkllV8psb/0d2ZvrGpfHxth1y6h1Xy1P+fLP8e/vuCT377d198n3X/FIu/OEN8tonnpd9yZR94q677KaYCfSRUV3y95/4uly1cc/ENbED8OpT5dXy1XXNMpPO7fO3GsPB1BbeTNv7Oh+J7L/p6SCwLEt2dQzKZx7dJH/+owflxWddK1cu/a788ClXyYf+tPaAall90aS8+5F18kNfv1Ue/Ylr5X/+/AGZy+v7vN/qPW3yE7+9V8668jr59fselqY5/h6JfE5es/7vcuodV8szH/zNuPLOsiz5VPdL8iMvfEN+Zs2VB11DOKCXykKIC4GPAyHgj8D9wCNSymMO3gSBEOIG4A9SyjVCiA8Bs6WUP95X+EVLF8uTbv04u1PtzAo18uFJp3FMZAGaYr+Jj+ayXPHSYzzcuo1St5eLZiziI9MWMKN0xEM1LYu7127kpmdfIpbNMaMqwjnzZ/P+2dOYVlmO+PGP4YoroLcXGY0iZs3i9rM+w02RBShCMH96LRemmzn9J98i98STeD/3WVi4EB54YCShS5Zg5fIo27byxPsu5EZlIblsgeu3/Ya5mQ4kkPMFcet5VGPi0b2HgimXP0TL1eccsvh03WCgN0lPV4zujigdbQO0NffTuqeXzr2DgN2Fb/rsWo49cSannrWIuoaJZ9Y0LYs97QNs3N7ButfbWLu5jUyuQFmJnw+ftpCPnbmMoH/sHPPRdJZndzbzty07eGFXCxL44MI5fOP0E4gERzzPWD7LX1u2cveODWyL9jG1pJyfrjiHZZXDE8GwO9XOgx3P8nTPy/T/vRltdYIXHv17EvgP3qaui8/hoLQ9c9EcufCmc9Etg/dXH8MH60+iMTAy0fTq7lYuf/FvtKViLIjUcPHMJZzdOIsSt3c4TE8ixTWPP8ffNm9HUxROnNHEmfNmctr6Nfg/91nb+34DPcEyPnj+FQT9Ho6Z38jyuQ2s2LKaqmuvQrS17fc75lU31086h2ciCxBCUD+5nMZpVUxqrKB+coTa+jKqasNUVJagucb2OBuufWTGj3o+EKZc/hAt15wLqgq33/6WetQNIaUkk8rbvYy6YnS2D9LROkBbcx/Nu3pIxu3utf6Ah0XLm1hx6hxOOG0uXp97wvhS2TxbdnWzYVs7aze38npzN1LC/Om1fOrcozhx6bQxHSCklOzo6efJbbt56LVtNA9EiQT8XHrK+7hw6YIxPR63xfq4b9cm7tv1Gkk9z4emzud7R79/WAe6pbOqfxP/1/4Uu1PtzC6ZwuWzP0O1L3LoexkJIb4H3CGl3DPq2Fwp5esHeqM3xPcpoEpK+TMhxPeB7VLKfXYdWL58uXzp5Zd4vHsN9+19kp7cAKWuECdULuG4ikXMC09FFSqv9HVwy5aXeGLvTkwpmVlawWmTpnNS3VSWVNbhUTUyBZ0HN23lLxtfZ2O73cxUWxLiw0aGf7/8K3T/6haq9DzKpZfCzp3s8oR55uWdvPhqM+3bW/nbvd/h6cZFrGzZwMMXfYnBT15CU32EKbXl1P3rJ1Eetqu11NRgXvMTurti1Fxx2TtqAIDixHUC63Of5/oLvsKlJ09HSnsUp1kcvKbrJnrBIJ83yBcHuWXTQ+MQcsNd8xIxu2tedNDudhobHDsJl6oq1DXYhcDUGdXMnFfP7PmTCJWMzPMupSSayNLaPUhzxyC79/axs62fHa29w2MSaiIhjlkwhZOWTeOY+Y1oxe6qfck0mzq6Wd/WwdqWdrZ09iCLv9O5C2Zx4bIFNJSXIqVkd2KQ5zubebp9F6u72zCkxZyyKi6Zs5wLps7Dpah0ZHpZPbCJ5/peYXeqHY/i4oya42i+fR2fv+RzTJs2bTjTvB1dF68/aG0/tuop7ml7lCe6X0KXBjNDjZxYuYRjIwup9VWQNw3+tPs1btu6np3xftyKynG1jZxaP40T6pqYEipDCEFzf5Q/rt/Ew69tpy+VRhGCzd/72sQj6oFt/30d91fNZc1rrSx55dlx42ompNgrKHHOBWx9rZ2dWzvYs6PHdg7ao2PGsgghKC0PUF4RpLQ8SGmZn1DYz4Jtqzjud1ehTDAtxJtx/YqP89VX/oL+vzdR+MiFI9oumBQKRV3ndHIZW9eZdJ5UMkcqmSURL3apHkwTHUgx2J8cN0YhGPLSMKWCKdOrmTarhtnzJzF1RvWYrtSGadHZF6e1c5A9HQPs2tvHjtY+WrsGkdIeazNnag0rFjVxylEzaKq3u8ablsXuvkE27O1kfVsHa5r30ptMI4BljfVcsHge58yfhdelkTMNXult59nOZp7Yu5M9iUFcisIZk2fxxXnHMD9Sg24ZbI7v5sX+jbzQt5GEkabOV8lFDSs5tfqooalr/vG7nQohSoDngaeAs4BjpZTxfYWfsbBS/u7xr1AfOJYq31I2xzt4qmctLw++TsHSCag+FpfNZGHpTOaHp+ETIf7Wup1H23bwcu9eTCnxqhqLK+pYVlXP4oo6FkZqsAqSZ3e18OLuVtbtaeORH3yDJ2cvIJLPMq+vm5tvu4eZ1RVMr4wwtaIcDYE6Zzb+vXbban8gzA2Lz+HxpmWsbF7PFavvxWONmnTP48XyenHHY/v6ageFBHuu+jdMdp1Qffxy0pk8E1nwtu/hD3goKfUTLvVTFglSFglSURWisjpMVW0pNXWlVNWE0S2LwVjGHqkcS9E7mKJnIEFXf4KuPnvwTjo7ktn8XjfTGyqYPaWKudNqWDSjnrJSP62DMXb3DbKzt98emNbdS3fCHhXrUlUW1ddw7NQGTprRxJSKMrbFenl1oItX+jp4ubedvqxtrKaWlHN6www+MGUOdUEfWxJ72BTbycbodjpz9uCemaHJnFJ1FCdWLiJnNNORWUN7ehXnT7n7kHU7PVhtT19YIe99+kfUB47FrTby995XeKZ3HbtT7QBM8lWxuGw2C0tnMDfURGsyxUMtW3li705ak7auavwhlldNYmllHYsitcwurWJHTz/P72zhYx89j8rB/gnvPZzzVRVMc79dSnOai9+d+1naTjmLmooSqiMhqsqDVJYGqSiz5/VRhaCnO0Z3R4zerhj9vQn6exNEB1JEB9PEo2mS8SyZdJ5TBl7jq20P4ZUH7ihJ3rrWFVUhVOIjXOqntDxQNFQhKqpKqKwJU1NbSs2kMkrCPpKZPAPxDP3RFH1RW9vdRW139Mbp6k+MGa9QHQkxc3Ils5uqmT+9lnnTaihYFnv6B9nVN8DO3gG2dveyvbufTLErcCTg5+gpk1gxrZETpjdiqZJXB7rY2NfFK33tbOzvomCZuBSFY6onc8bkmZw1eSYJM87m+C5eje3g1dgOsmYej+LmmMh8Tqs+mjmhanqy6+nIvIglLU6u++E/vkEAEEKUAacDz0kpu/cXdvaiOnnl/y2gYNn9gcvc06n2LabUM5uurJuN8R42xnYMD/MPaD5mhRqZEZxMva+GwTRsGYiyvq+D1wd7hmf/rPD6mVNWxayySqaXRDjx//0n4Q2voqbSrFp+DN8444IxE7d9dOurXHn/XWijquCm18umb32fmTf+nEDv+BfbowvuA8VUNcxgEFc8hlRUhGWSq6plxyWX0nniWViGiaXbE9xZhlUckWxvo2e3E8U5YFRNtUd6avaEX8rQJF8uFUW194WmgCLI6wbpnE46myedLZDK5Emmc8RTORKpHLGUPW1FJjc+I3vcGrUVJdRWlFBfFaa2Kkwo7MUbdJOXJp3xJB2xOHujcdoGY3TFk8PJVYWgqaKM2TVVzKqOUFHuR3gELakoO2J9bIv20ZwYHA5fHyjhqKoGFldWManERcKKsTPZxvZkKz05e2CQT/UwPzydJaVTmB5U0c1WerOv0pt7DVPmEahU+xZxZsP/HtJxCAej7RkLq+R37p+JxEITPqp8C6jyLUIVDexM6bwSa2ZLfA95yzaudd5KZoYmMyM0Gb8Spj1eYENfD+t624fnfVKEoKmknNmllXzgxQ28/wc/Qcvl9peMfSKBdGUNz1xwCU9OW0ZnX4KegQR5fXwzVMDnpizkIxzyEQ76KAl6Cfk9BP0egj4PAZ8bv8+NR1PBklQ+8gAzfv4jXIn4m+YRCew85Xw2fPLr9nQvxUnthnStqPbEjIqqoroVFE0d1rVQBRaQzetkcoVhXSfStrYT6Ryx5Mi0LBPNdVQW8lFT1HVdZZjKiiDBEi+KVyGWz9MRS9ARjdNW1HYiN2rwpsfNrOoKZldXMrWmjFDIQ4o8u+IDbI/2sTXay2Debp7ShMK8SDXLK+uZW1FK2Cdpz3axI9nGjmQrWdOOt9obYUnpTBaGK6n25IkVXqcnu5FYoRkAj1rK5MAJrKj51pFhEA6GyfND8kt3ryCV9uOzNCaXaaieHgxpP0SPWkq5eyYebRIxPUB71mJbMkFLugerOOOlT/XQGKil3luFKv2k8grdqQJ7Yil2xQfJmwbX3PonLnreniEwXhLkz5d+jt1nn4Fqqug5k8su/iTlfX3j0rffye54c4NQECoZt5eSfHp40rvHm5Yd9HMaWvgGim/ZRk0E9lZQhCDodxP0ewkFPQQDHnx+N16/C49XQ/NoCE1gqWBikdQLDKQy9Kcy9KXSRDPjpziIBPzUlZVQGfYTDnlw+1RwQVoW6EjHaUvG6Ewnhgt+AUwOhZleGqa+xEuZT0FR8/QXBmhNdzFQGHG+Kz2lzAlFaAq4qXDrKPQymN9BUm8fjs0nJtHWJ4iaOm5fnlJvlq/Of/qwDUybPD8k//W2kyjkvJRrGg2VFlmrgyHllLgaKPPMxJQV9BVcNKcLbE32MVBIjPreZTT6a4m4yzEMF9GspD2RY1csxt5Ugg++uIEbbr73LQ0q6wiXcdpXv0O530dlKEBFMEAk4CfgcuFCQZggizOX5rMGuWyBTFonlc6RTOVJZfKksvn96nD0/Ehxtx8EhPMZLKGgSovuQBm/Wnw2T0yz84SUvKVpr0fjdqkE/R5KAl6Cfg+BgBtfwI3HZ89cqnhUW9uKJGeZxLM5BtIZ+pJp+lLpcZPZaYpCbThIbVmI8hI/waALzaNiqCZRPcveVIyWZJREYcRQ+DQX08NlTC0NUh10EfBIDNK0Z3tpS3eRKzoBqlBoClQzOxRikk8hrGXIGXsZyG8bdpJdwo+eq6ItZpBTdUL+DImcn6uOfQ/OZVQ7p0x++/7FhFwjD7NgKcTzPnRDQ7MEIbeJz5dhaMkMgUrIVY+qVJG3QkR1ja6cSUsmS2++wFAxraBQ6Snj1Cd3ctEP78WVH/F8cx4X/3XJh/jz+xYDsOfT/7XPGU4HKiJU9I2vmg/6/PgMHd+oUaMFRSHl8VKazdAdLuOWs85n9ftOwOvS8KgaLlXBNbSAiLCnxx5aPGT0ajdCwvDvN3Ru1IIIxdUHiytFMbK0FEN/24uISOxPU9oL5eiWScG0yBk66bxOulAgnS9MuHKVfSuJz60VPUI3fp8Lj0fD5VZAA0NY5KRO0sjTn0vTn3vjC0VJld9LbdBDhd9NiVfg1kwskSdlJunJDQ57yAABVdAUCDDJpxFxSwJKBhgkbexFt0biNgoB4jkXeQmqyyTgzhPURuIpWCq92RBXLPnbYTMI9XPD8vt/XjC84hlAUneTLniwDBWvgBJfAc018r1cSpCgNgkpykkZPgYKCnuzOi2ZNJlR0197FTfV3ggBNcS3zr6Ssu7Bg0pb3uPmtku/wJoTTkAvWORyBpmsTjJTIJHKkdONfa5zoAhBwO0m4HHhd7vxaipuVcVVXCBHFfbCOLYMx2u76NHYOoah5b1Hzo+eWU/Y7ytGr81jLw4lhxfKMaSFYVnolkXeMMjqOpmCTipfIGfse2ZfiaQkMGQ4XPi8tsFQXQJUKGCQsXRihSy92TRpY+w7CbcKdUEfNQF7KveAR6KqBrrMEtXj9OejoxbrkVR73Ezx+6n1KpS5dNxKEsPqJaV3IoeWA5IamZyfZEHBVCRul0HYncE1aqnThO6lO1XCj4954L1nEIaWGbz7qcd4pv9JPJFuIqUJIr4UYVd2zGIhBUsha7gwTRUhwaOaeF0FlFEPSxFu3EoESZi89JHSNS4+9XeU/v/snXecHVd5979n6i17t3dJq967LMuy5N4wxQSMKcF0EkNCQsILeUlC3gAmhJBQAyTB9GIcMBiMbcC44m5ZLpIlWb2stve9e9vcKef9Y2abtJLVVzLnu5/5zN17z5w5M/Ob85z+tE/sNOevf/EXJPQkX3zjl6nqOLw/oK26nC+85Sr+5dt3jvORkLNM/v69b8TWdf7Pz35HXW8/3VWVfP8d1/PoJevRgtClmfRluPa7L0dWePQ8GS3pG3aaeX6A50u84eaqKLc/ilvi8YjhjufwgNA3gsDQtJGFxMzID0K4trxA16MquT68drwkEBJP+rjSJx945LwiWa94iBMeGfqa1iQxQ1IRsyiLGZRYGnFTYOoBQvPwcMj7eYa87EhNTiMgprmUm1BnW1TZGqVGQEIvoJHBl324wfjn5LgmBc8IF47WAizdI2GMf8lznkmvU0JvuoR0TxVT/WX8w3XvJRazJ92F5kMPP8K/3fNdOuytlFX1UZ0aojKWHXH5OvYaHN9ABgIdSdzwsMzxYUwthS6q8Ckh79ukPY0eJ2D6nVt51+ceJeYcOfMbRgJdtRX8943XcPuaJTi+wA9C15dj67u6EJRETnNimoGlGRiEPj6EFBCE61hJn2hww/AS7MHIUuzDn10/XMo6TEBkZo5R23LEMIQHGJFfBFOPNK5HS2obYsTXh6aHuhaRZ88AGTrCkT5F6ZP3XbJekdxhg0FCXetaQImlURGzKLV1kpGjJ0MLkMKlKAtk/Sw5f7SpzhA+ca1IrW1SGzOoMAUluktMC70DFv0eAkbPFwSCgmvj+BpSA0PziRvuuMJDICHtxunJldA7UIbbV8+rGl/NDRdfHt6Sc6FT+Xg5mt/Zl/Ye4NtP386AvZdExQDlJVnKY6GvWVsb387pSygGBn6gIWX4Upm6j6n7vGv+kxNmrlLAd3a8i4Jv0HTnHq791GMjLjUBCrbB1//uch66egGX3reD93zrSaq7huitLeVnH7yKp65eiYx8zvq+wA0krg+OF+B4krwXUPD80L3mIf5mwy36jBhT7T4VK8rIqNIgI5/LcsTvbOhfWSIiV4K2HroPtA0NWxdYhsDSBIYeutMUInSVGeDiShcncPDlcE0t9KdsCR9L84hpPuWWTqmhhe4ydR9bc9HJA1l8ebhR9gNB0TfwpYaMnKSbmj/uxQifr2DIjTHoxBnIJhkaLEUfqufqqVfwpugFmYjJNghH0vZPHryPRzr/QFDaQaosTXkyS5mdJ2U4aIeI1Q003EDHjzJjXYSGUR9TEAr9gx8k2eaEpeoJ9O7EdL778fU8fPV8ioGOK40Rd6+60LE1GxMTTZgIqYeO5gOB5wtcX1L0JY4vcbyAvBdQ9IPQL/gYfYeF/0P1DadK1zBcGZbj9K2J0f+1kQKLTkyPtG0ILF1gaBJDD91pIgKk8PCliyuLOIETnUGiE2Bpoa5tzSOpQ7mlkzIEST0grod+xHWRIwiGCDh8gmHRC/1T+whE5LbT0rzD/Cw7vk7ajTNQSDAwlCQ/UEGVO4sPrnsbs6Y1HvFu/NEZhCPh+z4/f+RhHmt/ilysHTuVpiSZIxXPU2I6JA0HWx/N2I/mnPy2B9aga6Gj9/ClaibZXiTbYPHc/2li3+trAANBDImNjzXihLwYaDgBFAJBwZfk/QBXCjwZOtwONzHmc+hgPoi+i7LpcWkSkY9lIcSYSnf4y3jkiHjD2ne0rnz0NzacRvjCaEh0EYQvjAjQCcJ9tBnRFtcFMV0Q0wS2LrGExNJCZ+S6cNEoAgXkBC/ByDMKwmsMIoMnhEQjvM8TOR73Ao2cb5FxbYYKMTL5OPl0CVq2mjnxxbz/suuoLC89/MCX4Ww1CEeiq7eP7/7hLvYWtxMke0mkMqQSOVJ2gaTpkDCKGBPk9oEET2oEUmP2XV1c8OX9lLQ7SB2ED9nGsXoeRRADbAIsfGniWCYtSQAAIABJREFUSgM30ChKQcEXFAJJwQ83T2p4Y7TsSw2fUZ0HY/QdHOJRQQz/CRF5Wh7+Fo6u7XBSVTDSSDQ27cPajjTNGG0fomtdBFhCEtc1YjrENIGlSSwtwNR8DOGh4yJwkBSAI9e2PH/4XY7SIUJfy4Y2cX5b8A2ynk2maJMpxMhkkzjpUkqcRi6ftp7r1q1H1/UJjz0ayiAcI1JKntm2g7u3PUSX18Kqp5/m3f/zMLYzaiTcmMZDN8/j4OurMLSjj5n2h0tAY0rywyVwLapmnjiRY2P0aB/KPCTcH2o0RglNyvDn0LtCuB+/nRiBJHzJw/ao0W4MMcbIHKXgJyUUApOCb1DwTPKuRdaxyedjFPNxZLaMMlnHBQ3n8ScXrsO27SNHdoKcawbh5cjnC9zxxCM8172JQa0T4mnseJ5EvEAi5hA3isQNF1t3x/kmn4jw+YaZt5Rjs+bwuYYZ7cmkVhu3yUjfo2c5shueUV0HjOp8WNv+yPcngpzguke6LqLahj5RFWsMXqDhBAZ5P9R1vmiRi7TtZpOYTgX15nSuX3YpS+fOPWyF31PB8Wr73PCpfBoQQrBm8QLWLI6Wj/1T4Pxbw3WFmpuhqQnzs5/lmhtvpFBweOLFbWw4uIl2p52cGCCwMmh2AcsuYlouMdPFND1sw8PSPEzNx9J9DHyEdvgQveNDAj5S+ofJO5BH6tYbE4bRKvRwbMN9zCdP+NK4UscNwq0Y6BQ9g6Jn4LgmRdek6Fh4BRuKCWyvjCqzhoWV83jVqjVUVRx/qV5xZOLxGDdeeQ03cs3Lhu3tS3PPc0+ys283/V43BWMQYeUxYqG2LTPsp7AML9w0P9S25mGIICrHn0xhJ+r2HT9ieqQZ6WgSHS7GDDefyZH/TyI5EUKEafClFmrbj7TtGzi+QdE1Ql0XTVzHxndiaMUSErKcxngDF05bwdqlC7GsiWc1n6380dYQziT5vEN7ZzdbWvdzoL+dnnwfGT+NE+Qpag6B5iA1FzQPzXDD0rXphR2zuo+mheMQDMMfydSFAEP3Oab3MXpBPF9DBqNvi+/rI30VnheWDXzXACkIPBMCDXwbPTDRpUWMJEm9hHKrgoZUDfPqm1jQNI1U6mUWLjvLeaXVEM4k6XSWbQeb2dXeTEemi8FimqyfoUAGT7gEWhH0IugBmh5qWx+jbRG16+tjmm81TaLrUXZ/LNqW4Pr6SFiJwPN0BKHGgyDUfeAZSKkhPQN8EyENtMDGCmxskaDUKqPSLmdGxRSWTJ1JY30Vtn1uZeiHomoIZyHxuM2sGVOZNWPqZCdFoTillJYmWbt4IWsXL5zspChOAce6/LVCoVAoXuEog6BQKBQKQBkEhUKhUEQog6BQKBQKQBkEhUKhUEQog6BQKBQKQBkEhUKhUESccYMghDCEEM1CiIej7eTdfCkUZwFK24pzncmYmLaM0An5xyfh3ArF6URpW3FOMxlNRmuB1wkhNgghviOEmNAoCSFuEkJsFEJs7J7AS5lCcRaitK04pzntBkEI8c0xVeiHgRrgKinlGsAEXjPRcVLKW6SUq6WUq2tqaiYKolBMKkrbilcap73JSEr5gbH/CyFsKeXwIvkbgbmnOw0KxelAaVvxSmMymox+JIRYLoTQgTcAmyYhDQrF6UBpW3FOMxmdyjcDPyFcuPbXUsr7JyENCsXpQGlbcU5zxg2ClHIL4WgMheIVhdK24lxHTUxTKBQKBaAMgkKhUCgilEFQKBQKBaAMgkKhUCgilEFQKBQKBaAMgkKhUCgilEFQKBQKBaAMgkKhUCgilEFQKBQKBaAMgkKhUCgilEFQKBQKBaAMgkKhUCgilEFQKBQKBaAMgkKhUCgilEFQKBQKBaAMgkKhUCgilEFQKBQKBXAGDIIQok4I8eiY/00hxF1CiMeFEO873edXKE4XStuKVxqn1SAIISqAHwDJMV//NfCslHI9cIMQInU606BQnA6UthWvRE63T2UfeCtw55jvLgP+Pvr8CLAaeOjQA4UQNwE3Rf86Qogtpy+ZJ0w10DPZiZgAla7jY/4JHKO0PTmodB0fx6XtU2oQhBDfPCQBD0opbxZCjA2WBFqjz31A3URxSSlvAW6J4t0opVx9KtN6KlDpOj7O5nQdQxil7bMAla7j41i0PZZTahCklB84hmAZIA4MAiXR/wrFWY3StuKPgckYZfQscFH0eTmwfxLSoFCcDpS2Fec0p7sPYSJ+APxGCHExsAh4+hiOueX0JumEUek6Pl7p6VLaPv2odB0fx5UuIaU8XQk58kmFaCQsSd0rpRw84wlQKE4TStuKc5lJmZgmpWyTUv5MvTCjCCF+IoT4whF+2y+EmPEyx79FCNEihNglhLj0FKXpYSHEZacirj8WlLZPjJPVv+LUMBlNRoqJuRzoOJEDhRD1wNeB9YSdmncKIeZIKf1TmD6F4nRywvpXnDrU0hUniBDCEEI0R6Xoh4UQS08irsVAM1AuhKg6gSiuAx6RUu6SUm4GcsDCE03PqeZU3ivF6edMP69ToP9J4ZWo67PaIJzlSwMsA26TUl4WbS+eRFxXAg8DjwKXCyE0IcR/CyHahRC3AuZwQCHEp4UQrZEQ3xl9vRDYF/3+nSj8u6L/HxZC3CCE+JUQ4sEx8XwwiqNVCPHxMd//c3Te3wKlJ3FNYzmV9+qUMFZbZ1pXZ7mu4cw/r2PSvxDiO0KIg0KI9Fj9R01KP4nCf04I0SWEeNdpTjOchbqGk9P2WWsQzoGlAdYCrxNCbIiEejLNb1cAf4i2K4EbgFXATOAOoBFACNEEXAzMi87/H9HxZUBOCHE9oEsp5wHVQoi50e//CnwXeGMUTwx4ZxTHXOBjQogSIcQa4H2EI2T+mXDo5KngVN6rk2YCbZ0xXZ0DuoYz/7yORf+vIiyg7AJ+Bfwpo/oH+A3wOFAD3Ew4a/x0c1bpGk5e22etQWB0aYD0mO8uA34WfR5eGuCMIIT45piq4cOEwrtKSrmGsATzmhOMVwcuAb4H/BvhC7EO+LmUsiCl/AUwACClbAb+FvgocCujM2GLgA28Gni9EGIAaGN0TPx3pZS/Hu7olFIWCGsQ7yS8n5WEU+/XAfdIKfullM8Ap6rE8wyn4F6dQg7V1mWcOV2dVbqG06ftYzz3sep/bRTmb4EY8G3GzwR/knAi4PD+TORtZ5uu4SS1PekWbRhxCpcGOB0cOlNVCGFLKZ3o342EJe0T4Txgn5TyvCjeNiAFjB0PHES/XQx8H/gkYSlgf/T7viieDGHn3JcI+xGG789Th6R9NqE4PkVoXJYM/zTReU8Bm0/RvTolSCnTAGO0ddp0dbbrGk6rto+FY9V/AqgFvgbcDuwFFowJ4x+yPxOcVbqGk9f2WVNDkFJ+YExb3GVSypsnCDa8NACESwNMZvp/JIRYHpVw3gBsOsF4rmD8BKanCTPr64UQthDiTwhL8AAXABuA2xhfGvkVYZXaJKxeryAsRR3p/qwkNCbfJcyspkbfbwBeLYQoE0KsImwjPRWcqnt1ujhtujoHdQ1n9nkdq/6zhM1IGwgz3wWHRjQJnO26huPU1mQL73g5m5YGuBn4EfAC8KSU8v4TjOdKDn8hrgJeIhx58T5Gh+P9nLA030bYzp8RQsyTUu4E/hJ4LWFVeg8wiyPfn+G0dgJvI6xhzJNSPg78L7AT+Aqw7QSv6VBO1b06XUy2rib7/IdyJp/Xsep/CzBEqP/vE2ZyGSHEvNOYtpfjbNc1HKe2JmWm8vEghHhYSnlZ9Hk6YefR/YTtjGvVWPtRhBClhCM1HiDsT1irJkgdmWFtTYaulK6PD6Xt4+NEtX3WG4RDEWppgKMSjTK4mnBegproc4xMtq4m+/znAkrbJ8bxaOucMwgKhUKhOD2csT6EyZwIpFCcTpS2Fa8UzohBmMyJQArF6URpW/FK4kzNQzjU/+xlvIzvWTHG72wymTxvwYKzYZSZ4pXIs88+2yOlrDnBw5W2FWctx6vtM2IQTmSyxFi/s6tXr5YbNx6Xa1CF4pgRQhw40WOVthVnM8er7cmah3C2TcRRKE4VStuKc5bJEuvZNhFHoThVKG0rzlkmay2jE/E9q1CcCyhtK85ZzmgNYXhmppTyAOEEk8cJVwtUszIV5zRK24pXApO22qmUso3RZVkVilcMStuKcxXV4aVQKBQKQBkEhUKhUEQog6BQKBQKQBkEhUKhUEQog6BQKBQKQBkEhUKhUEQog6BQKBQKQBkEhUKhUEQog6BQKBQKQBkEhUKhUEQog6BQKBQKQBkEhUKhUEQog6BQKBQKQBkEhUKhUEQog6BQKBQKYJIMghDCEEI0CyEejralk5EOheJUo7StOJeZLAc5y4DbpJQfn6TzKxSnC6VtxTnLZDUZrQVeJ4TYIIT4jhBi0jy3KRSnGKVtxTnLZBmEZwj9za4BTOA1hwYQQtwkhNgohNjY3d19xhOoUJwgStuKc5bJMgibpZTt0eeNwNxDA0gpb5FSrpZSrq6pqTmzqVMoThylbcU5y2QZhB8JIZYLIXTgDcCmSUqHQnGqUdpWnLNMVvvmzcBPAAH8Wkp5/ySlQ6E41ShtK85ZJsUgSCm3EI7GUCheUShtK85l1MQ0hUKhUADKICgUCoUiQhkEhUKhUADKICgUCoUiQhkEhUKhUACTN+z0j5pC3mFPazv7u9poS/fQnesl5+XI+3kKMksgfDytgEQiNRc0H5BgeAgho1gkmukhkEc71TgCz0DK0TKA9AyQGkiB8C1AYPgxBBq2SGAJkzKzjFK7lOpEJU3VtaycNRc7bqLr+im9J4pzH8/zyGcdNh/Yy4HuDrrz/aQLaYa8NK50KcgcEh9PdwCJ1IsgJIgAYXgj8QgtQNO9I5/oECSCwDUIR/qClAK88H8Z6GiBiZAaRhBDR8cWCRJ6grgRpy5RRWNpDTPrG5nZ2IAds07tTTnH+OM1CLfeCp/4BDQ3Q1MTfPazcOONRz3EcYps2r2Hx/dsoiXbwpDoxzeGEGYRI1bAtFws08UyPCzDwzR8TG10M4SPoQXoIkAXEmrAqoEpZ+iST4YisBvY2Qa+1PCkhid1vEDHDTRc36Do6+HeHd4sPMciKNpoboKEX061VcuyuoVcuWIlJanEZF/WK5MT0PbgYIYHNz/H5s4d9Ba7yGuDBFYWzXIw7WKkbQ/LcDENH0v3xmg7QBc+hgi1rcWBJign3M52uoAuH55oFqG2Aw13RNs6rq9T9A1cT8fxDIpFE69o4jkxpGtjuClKqWBqaioXzVrJ0jkzsKxz07AIKY+9hDlZrF69Wm7cuPHURXjrrXDTTZDLjXxVtEx+cNPVPH31VOxEjljMIWE7xE2XmOES011szcPQgqNGLSX4UiCJNjlSbkEAmpAIIdHEqbucM4mUEETXF0gYvjoAIYavMXjZ6wskOIGB45sUfJOca5EvWhQKFvlcnCBfQqxYSVNiFm9cfhnzZk47bdckhHhWSrn6tJ3gKJwJbTuWyQ8+eDkbXzWVWCJP3HaIW8VQ23q4WZp3DM9MRM8+KoVHCOSYZy8R56i2Axle1/A1jmhbhNeoCYnGy1+fG2gUA4O8b1LwTPKuRc6xKORjFHMJyKcolXUsr1rEm9ZdTklJ/LRd0/Fq+xVtEKSUPLppM3dtv58+vQWjZIBESY5PvuUuyjvyh4XPNFr84uHzomPBkxq+1EYydYFE1wIM7ej3TBBDYiMx8aWBJw1cqUVCETgBOIHECSTFQI6cJ5AavhQERHupESDGZMDDhiZ8OYERo8MEjUcjIUZ0HZkpQSRsOW6viwAtEr4ugpG9QYCpga0J7Ghv6RJLSEwtwBA+uvDQcBE4SBzAP+L98QMRXW+YZiHCjMSIzjkRed8k61lkizaZQoxMNoEzVEKsUMeqyvN4x+VXY9v2UZ/LEZ/XOWgQ8vkCP/jD79jc9zxOvJtYaohkMs8/vukeytsLh4UPdHjs83PY9/oafDnm/kfaHn7W+lG1rUfatghGtK1TDDTcQFAMiLQNhSDACwQe2si5RvaE+wARZsCH7CXDxSeOSdvDv2qRURrOvEcz8WEth5/14WuNtlDXkbZ1gSkklhZgiGFtuwiKY7R9ZLxA4AfheysY1nYY10SGxJeCnG+RdW0yRZtMLkYuk8TPlFEZTOcNi69h3dJFRz3n0fijNAhbdu/nextvZ8A6QLxsgLJUhrJYnlIrT3yCtsh3zX+SifIdKeCHOy4c841AE0kgSUAMJ7Ao+DoZH4ZcyaDnk/c1ioGOK41orzMsV0MYWJqFgYmGAehIqREEAj8ALwDXB9eXFH2JG0gcL6DoB/hRaUUO76OMX0YJPbSUdqTHOFaEAhmVdqLSvACGS3ditPZiaRqWrmEZAlvXMDWBqYOhCXQ9fLGECEAESDx8PFxZpBgUo1dTohNgRU0KluZhC59SU6PM0EgakNADbM3FEA6CHIEcQk5gRIpeWHUPomvRtWDC0qwbaAx5MQadOIOZBJnBMvRMHVc1Xs6bLrkccZRi3dlqEHzf57aH7+fRrkcJUp2kytKUJbOU2XlShnNYbTWQ8J4FE2sbQu045QYb/mkG+1/fgCZKkCRwZQwnMMj7GhkP0l5A2g1wAp3isK4DHZ+w30gAlmZhCgsdA4EBUkdKERr7sdoOJI4nKfoBjh/g+sFo4WZE46Nt/zJK6LBBCL+f+HrGaTsyBgwbhbF7Ee51AZYusA0dUxNYusA0BKYGRrQJLQi1jU+AjyddirKIJ4fzEYkpfCzNxxIeluYR1yVlhk7K1EgaAXEtwNKK6OSRZAlk9rC0BwEUfSMscAKaJjE0H2uCFoi8b5IuxhgsJBgcSpIfKKfSnclNF76FudOPXnM+Xm2fU30Inufxlbt/ykveBhJV3VSVDVEZy1JmFlh23mg4N9Ao+CYF18QpWliaj20WR0o/2QaLkrbiYfEP1STZMrSCLqdI1jcoBAYyGoilC42kniSmxTGwkNLA9QSOB9liQLroM1BwyXsBvq8RBNo4UQMYQiNhWMR1A1szMIWOITQ0BAkpEFIgAyCQBAEEgUQGEt8PCPzwOz8ICKTE9yVShlsQyKj55nA0AUKIKDPVEJpAEwJdF2hauOm6Fn0GoQmERljA98JMxBcBjgzwCHACj4LvkfNdHN8DxraVSjQtrEVZuqTMNimzDZKWTmCC78Og5hEIl2JQIONnKQbuyLGW8InrRaotgxrboNwUlJgeJVoeQQYv6MWXowZeSnBcE8c3CIRA1wIaEoPMTPVCw0FgC1ke4Js7TPqcBH1DKQb6KijNzeFjl7+HxrqzZ6XR/S3tfPnRH5BN7qW8qp+qkgyVdpZYk8e6ptFwed+g6BsMFWPoMsDWfWwrvIeaOLK2IczIYwMeaz6xn6cGZvKHq+cAYGsWST2BJWJoWPiBjusJ8m5AphiQdjwGHBfXj0q/gYBDtB3TDRK6SUw3sTQdU+joCAwpMKUIS1uBRI7TNQRBQBBI/ECGGpfD+yBqnpRHNAiaAE0Tkb4jTQuBrmloukDXBGKMzjUNCED4AikkARJXSPIyoCg9nMCn4LtkvSL+IScVSDQtQNcDEqZGmW1QahskTY3ABMcL6HI8fIoUgjwZL0dAEB0bENM8SgyfWtukytIpNyUJ28UUWaQcpBj0h4mL8H2NgmfiBhpSQMIoUlWWxajohCaA53jc+QW/3RajN19C32CKXG8ty+wL+NBrbsAwTixrPydqCLXzquRf/3QttfEhYmNK/AXfIOPauK6OKQUllkfMzo5p+tMpMRvRRA2OLGHANegsBDTc8Tzv/Nz9xJwxmQvgmjp3ffQGnnjDpeR9g8GCT3fOpTWTJ1v0GfsSaEJQaSeotOOkDBtbGOiBBgH4xYBi0Sdf8MjmXDI5h6F8kbCAfuSSqqXrxC2ThGlimzqWrmNqOoamoQuBJjS06NwCMTJmeGycw59GxyLJ6FsZVcUhIHrxkPhBgCcDPD+gGAQ4nkfB9cgXXXKuy9GQSGK2QSphk4ybJOImtm1gWhrCgECTONIjF7gMFPN057MUg/G1AFOT1JfEqEvGqIjpJG0wdB+PAhl/iG6nH0/6I2dM6YIZyTiNMZMqKyCh5xCyn5zXiitH2809N8aQY+EEAs3wiZsuKXO0uu9LQa+TpDNdzhcvun3Sagj1Cyrlx25fTZWdHddclnZt8q6F9DViQlJiFzHM0aYgUyshYTQSUEHOj9Nb1GjLu0z51Qvc9PkHx2l7InxN4zMf/lN+fN5ivGC8Jm3doCaepMyMkdQtLHQ0XyB9KBZ9io5PLu+Szbmkcw5O0TuqrgUQN03ilkncNLANA1PXMSNd6yLcCyHQoni06EgxJo5h5LjPcuS7IPrPlxJfhgUnNwjwAp+i5+N4PjnXJV90KfpHbtKUUVNUScKmJGGRjJvEYya2paObAmmAJwIc6ZHxHHqdHH2F3CHNWpJSS6c+FaMmYVFqa8RNidBdijJPf3GQfjc95h4FNMQsmuIx6mIaZYZLTMvgBd1kvfbR2rMU5AtJMq6OJ8A0PUpMZ1y+mPdNunIpOvsq+K+rf/zKazKasSQlP377KnryJRQLFimhM6USfK0DGVnVpFFHhTUXKWroc22acz47hgZpK/SNxJM04jQl6qm1q1n062e46Cu3UdbZS1d1Bd+6ej2XbnqJi7fu5ul5M7j5A28jmDuXaSVllBtxDKnhO5JcziM9VKB7IEv74BB59/AXryIRpzqZoKokQUUiTtwwMNHRJOBKfC/ALfgUHRen4FLIeeTyRXK5ItlCkWy+iO8fvfP6dCEExG2LZNwimbBIxExiCYtYwsSOGZiWgW7paKZAauARkA880nmH3mwu3DI53GB8+nUhqC1NUleeorI0TjJpYsV0fD0gGxRpy6VpzgzQnR9fvZ5aUsqc8hQNJTHK4wLDKDLoDdKc66Db6R8JV2okmJ+qYlYyRq3tYYsBhtx9DBT3jbxMtqimq9+mr+gjLY+KRJaUWeDP5j8xaQZh5tIS+eHb1jKQTyCKJtW2oKq8QFGGutUwKLdnUmLOpBCU0VXQ2Z112JnpIeON9oPV2ZVMS9STMsqYe+dTvO4TX0cPXmYARLQfqKvlN+96J4+uuYiBdJ6OgQxdQ1mCQ/IGU9dHdF2VTFAas4lpOgYawofAC/CKAa7jUSx45HNFnLxLNl8klw91nXfcI5b4TzeGrpGIWyRjFsmkRTxuEY+b2LFQi4ZtoJsaGIJASFwCcq5LXy5PXzZHTybHQP7w/pmEadJYnqK6PEFpyiaeMNEtgSt8BtwCBzMDNGcGyHmjBSxbN5hTVsGMsgQ1SZMSOyDQCnQWemjOdZDzR88zNV7N/JIUTQmDCrOAlJ30F3eR80LnSgIdza+ntV+SlT52vEhVfIgBJ8E/rLj3lWcQ5iyZIr/629fQU9gGSAwRoya2hJrYEhCN7MkGbBpsYevgXpwgrC7X2BXMTTUxJzmVhF5Gx5DH1p4+nutuozkzMBJ3QyLF/Ioa5pZVM6OknNm338XK//gimuPwv6+7nq8uv4DMmPaYpGXRVFnG1PIyGstLaSxLUWrbBIWAQrbI4GCOzp4M7d1pOnvTdA9kJ8zcTUOnPBWjNBkjlYyRStiUJGyScYtELMyQY7ZJ3DYxdQ0hQQYSEYRNRAQS6YfVbaLfwg/D3W1ypKlIaFrYDCQEQg+r01FbElIDP5AUih55J3xhh1/gTL5IJueQzhZIZwoMZgrkChM3R1Sk4tRWpWioLqWhupTyigTJUhvd1ikS0JEeom0wTUt/moP9g3RnRjN+TQimVZQxv66a2bVVVFXEsBIG7fk0uwZ72TnQzd7BPjwZ3scyK8aK6kZW1NTRVGZjGA77c63sGjrIgWw7QdSlNzPZyIry2SxIJSg1BulzttGZ30Te7wmfpVFPhb6Cq6Z/ctIMwrxljfKzd54/8nInjTpq48uptBfS76Z4KZ1h0+Be9mfbgLDpckaykbklTUxPNFJ0LQ4M5nmhu5MXetoZcsNa0BuffIHPfe+XxJyJn9eh5E2T/7jhney86lVMKS+lsbyUutISbDTcvEcuXaSvL0t7T5rO3iE6etMMZg7PHAGScYuykjilJTFKEzbJhE1J3CIZt0nETRJ2qO2YZaBrImwpCSQEATIA6QcgJXKkWTQcIDJSXxBhrVdoYYeYpgmEpoEefid0DaJWqqLnk3c8Co5LLipsZQtFsjmHdNZhKFsgnQ217XqH1xoMXaOmooT6qhT11aXUVaUorYhjJ010W2fQcWgbSNM6kKZlYJCD/YPkiqMZf8I0mVNbxbzaKppqy0mlLDwjYP9QP7sGe9jR301nPjMSfmaqgpU1jSyurqSuRGfIH2B35iC7hprpLQ4CENdtFpfNZnnZFGYlBIFspbvwIt2FLfiyiECjJraEFMu4uOlDrzyDMHtpufzPe25gSnItjYk1OLKah7ue55Hu52jLhy9SU6KeZeXzWFI2m3klM9jVP8jvmnfwUOseWrNh1awmnmRVzRRWVDeyrKqe+WU17Ovq5/E9B9iwv4UX2zpxfZ+aoTSfufdXXLb1BdrnzKXrmlez4M5fYLW14U+Zwrb3f5iH569h14Fudrf00Dc42lShCUFtZQkN1aXUVZdSV5mipiJJdXkJVWVJKssSVJQmCFyPrvZBujvT9HSl6esZor83w2B/jsGBHOnBHJl0gWymgFM4etPNyWIYOskSm5LSOCWpGGXlCcoqkpRXJamsKqGqppSa+jJq68soKYsxmHXoT+foHcjSO5CleyBLV98Qnb1DtPekae9J4xRHa04xy2DW1CrmTKth/oxaFs9uYEpdGa2Dafb29LOnp5ddXb3s7OzhQN+osZ5ZVcH5M6Zy4cxprJ4xhU4nw4u9HWzqaee5nlZ2Dgxn7BbrGqZzzbS5XNI4g45iB1sH97B5YBcvpffhSZ+kHmdt1VIuqz2PWckEnfnnaM09RXvuGd4x96FJMwizlpbdoMGtAAAgAElEQVTK79z7V0xJXkBtfBW7M0M81PkMT/dtIe87mMJgUdkslpXPZUnpbGqsWh5q3cd9B3fyREczec9FAPMralhVM4XlVQ0sqaqnxkrS8vX/Zt6/fobk0NBRGnRGkbrO0x/7NPfNXs2u5m72t/Ud9hwba8pGMsbaihKqK0qoqUhSWRpqO5WwGRrIRdoepLd7iL6eIQb6sgwOhNrOpPMj2j7dNWE7ZlKSilFSGqe0LE5ppO3KqhIqq0uori2jpr6U2voyMDT60zn6BnP0Dmbp6c/S3Z+hq2+IjkjbXX1D42o41eVJZk+rZl5TDQtn1rN4dj26rbOvt5+9PX3s7gq1vb2zm8GodmEbOkun1HPBjGmsnz2dadWlbB/oZnNvBy/0tPFcdyu9hTBPaSop54qps3lV0zxmlpWwY2g/Wwb3sGlgJ635LgCmJeq4pGYVl9WsRBPttOeeoSX7JIaweU3TN195BuG81avksxufY8vgHn7afC/P9W9HQ7CsfB7rqpexpnIJNbEKuvIZfrj9OX66exPd+Sxxw+Sihhlc2jiLdQ3TmZmqAGDjgVZ+tWkb97+0m8GCgy4EixvrOK9pCiumNbBsSj31pSWIO+4geO/7EEPpcS9UXjf59/VvZe+l1zJnWjUzG6uY0VhJU30FDTWlmMboLN7sUIGdL7Wxe3s7B/Z0cXB/D60HexkaHD/sVQhBaXmC8sokZeUJUmVxUqk4yVSMRNImkbSIxS1iMQsrZmBZYfONYeroujbSMTw8mmakw9mXeJ6P5wV4rk+x6I02VUXNVLmMQzYTvqBDg/nwxe3PMdCXwXUPafO3DBqmVjBtejXTZ9Uwa149cxc1UtdQPu7cPQNZDnb0c6C9n31tvew52MOu5u6RUmXcNlk+r5G1S2dwyXmzmVIbTmHKFBy2tHexuaWdZ5vbeLa5lYxTRBeCC2ZO47plC7h20TzilsmAk+epzmYebdvPQy17aMulsTSdVzXN470LV7OqZgoF3+H5/h081fsiT/ZsJuvnaYhVc/3UK7i6fi0aPoYem8RRRufJJzY8xe/an+CXLQ/R5fSRMhKsq17O2qplLCufS0y3eLqzme+9tJH7D+7GkwFTS8q4YspsLm6cyZq6aZRZMbJOkd9s2cFdL25n44FWAikpi9l84OBu3vM/X0V7mSYkCJuR0rEkv/yT9zNw3RuZOaWKpoYKmuoqqCxLjHvGHa397NzWxt5dHRzY003LgR46Wvsn1ExFZZKyiiSlZXFSZQlKUjGSJTaJpE08YRFL2GHTjRVqu/qBu5n6xU+jDw4clka/vIKuv7+ZgVf/Cb4f6totehSLHk4h1HY+VySfc8hG2s6kC6QHQ4M02JclPZjn0LyvtCzBlOlVTJtRzcw5tcye38DchY0kkqPDmYuuR1t3muaOfg609bG3NdT2npYevMi4NVSXct6iaaxfMZMLl84kHjORUtI6kGZLWyfPH2xj44FWXuroJpCSikScqxfO4Y0rFrFiagMAewZ7eaLjAH9o28tj7QdwfI+GRIq3zl3OO+evoiqWoLPQy9O9W3iiZxNbBvcAcH7lIt7a9CoWlM7AD5zj1vY5YRBWnbdKvvXHH+aR7ucoN1Nc13gJV9evpcouA6Do+3zjxSf4n61PU/Q9rpg6mxtmL+XyKbOJGeZIPI/tPsCX7n+MbR1dJC2LKxfM4soFc1g3q4lUbPwY9p0Hurn1txv5y4++nbrc4aKUU6ciDh48bFao/OxneWnpJXT8+9dZ/stvU5Hro9ss46nSuazL7qa6MEA+nkLXNaxsGr+0DE3XEH19CF0Ph+IcuhfiyGPvJuDL69/OR/pfgK1bT/COR9coJUPpPL1dQ3R3DtLZPkBHaz+tzX0c3N9DW0sfQfQSVNWkWHH+TC64eD4XXDyPWPzwmZpSStp70mzd08ELO1rYuO0g+9vCtvJlcxt5yzUruXLNPLQxY0o9P2BzawcP7dzLvVt30tw/SHk8xvvXr+bda1diRaMppJRs6m3nV3u3csfeLaSLDldOnc3Na65hSkmkk8DliZ7N/Lr1YXYMHWBqvI6PLXgn80qnT5pBWLxyqVz7zRvoKPSyuHQ2r59yCRdULcXUwus6MNTPPz11L4+276fCjvOm2Ut4w8zFLK6sG8mcC67Hdx7fyPeffI4hx2FmVQXXLp7H5fNmsbixFl3TJpyw9rJUVcFXvzoyyzmfc3jyDzt4+tEdbNq4n/7esKlD1zWmNIUZaeO0SuqnVFDXUE5NXSlVtaWUpGJHHfY7wq23wt/8DfT2HjHIl9e/nY88/hOwLPjud192BvaR8Fyf/r4MPV1putpDbbe39NFyoJfmfd0M9IVNmpommDWvnvPWzmb9FYuYt6hxwmtxPZ/dzd1s3tXGc9tbeHbbQYZyDjHL4Kq187nxNauZNaVq3DGD+QKP7znA/dv38NCOPeRdj+VT6vno1RezZsbUkXA5t8gDLXu4fc9mHmnbR8Iw+aul67hp8QUYWtj93l3o53cdT/CbtsdIe1muqruAD865gYRxjhgEIcR3gEXAPVLKfzla2NpFU+Xab97A25pexfVTrySmj2Y2ju/xvgdu5/GOA1w3YyEfXXEJM0orxh0vpeQL9z3Kd554lqaKMv78ovN53dIFxC3z0FPhBwFfu+0R/vfe54jbFg9++8NHXi9oxgxoaQFvtFrt6Ca/K1/Oq/o3EQtGm3qGx/qcCWZ8/G72f/51sGjRSRuFo1F0XPbt7mLH1ha2vtDM80/vZXAgR0kqxvU3Xshb3nMRpnn04W+tXQM8uGEXd/7hRQ52DLB4dj2f+/B11FWmDgsrpeSZA6185/GN/GHXPhY31PLtd1xPRXL8TM+sW+SHO57j65ufwNZ1brvm7cyvqBkfT99W/mv37Qy6GX518ZdOqUE4Hm2XL6iTN/zwQ3xo7ltYWbFg3G+betp5x33/C8DfLr+IG+etGFfAAegayvD+H93Brq5erlowm/etW83KaQ0TZlp9//0tgr//ByrTvRw+cPSIF4OUkmxFLbfUXsq9yYVUVJWwcs0sFq9oYv7iKUyfXYtlneAI9mMwAmMZ0TbA9Omwf/+JnfdlGOjLsOuldl568SCbn93Ptk0H8f2AOfMb+POPvIoV58886vGeH7BpZyv3PvES9z6xnaLr8+7r1vCBG9ZN+GwyTpG7Nr/Etx57hrbBIf76sgv50GVrDwu3e6CHf3/+D/z+4C6umjqH/7ns+hGjAJD3HX7W/Ht+fvB+5qSa+Mqqjx2ftoebFs7kBlwPfD/6/F1g7tHCl82vlRt7t8mJ+NbWp+X0H3xO/nTXpgl/l1LKB17aLed/8kvyU3fdLx3XPWI4KaX84d0b5Jp3fFF+/nv3y3QmL+X06WGf1qFbWZmUtj3hb8FE4c/gNv3jd4/+fwbxPF8+v2Gv/PRHb5PXrPpn+fn/94tjPtb3A3nPo1vlZX/2Nfmef/7xy4a/b9suufTmr8q/+8Vvjhhmz2CvPP9nX5Nv/M0PJ/x9wEnLP9twswQ2yknSds3CKXLIzU6Yvmvu/LZc9/NvyINDA0e8xr/8yZ1y5b98TT6yc98Rw0gZ3t+3fvz78qoPfkPe/9QOGfz4x1Lq+nHpKgDpllVI/0c/Ouq5XpYf/1jKqqqT17YQJ5eO4yA9mJN3/3yDfPfrvyKvXf1J+dzTe4752P50Tt58y+/kmnd8Uf7i/heOGjZfdOX/veO3cv4nvySf3Nt8xHDf2/aMnP6Dz8mf7Hh+wt8f635Bvumxjx23to9p+WshxFuEECe2LsDEXAb8LPr8e+CiCc55kxBioxBio+FrrDqk9DTM9v5uKu04b5q15Ign29LeCcDfXLF+pInhSLy4q43q8iR/e+OlpJKxcGGwxCGLsCUS8I1vQPHIE4DONF9e/3ZmfPxuZnz8boCRz1++b+cZS4Oua6w4fyZ//9k3sWDJVDY+sfuYj9U0wWsuWsS16xbw0r7OCUd8jOWqhXM4r6mRbe1dRwwzq7SSq6bNYXv/xGHu/dVv+eistx9zGo+RyzgObWuOpMQ4fJE/PwjYOdDNa6cvYGrU5DUR29q7WDe7iYvnzjhqojJ5h32tvbz+kiVcecE8xI03wg9+cLi2j4IAjMF+tHe+M2zOFCKsJd9669EPvPXWMJwQoGnwjnccc40AjqLt13zwmOM4WVKlcV77pvP5xL+9mSCQPPfUnmM+tjwV5+/edQWphM3mXW1HDRszDf72ivUAbG3rPGK4G+evJG6Y7BjonvD3C6uW8pmlf3HMaRzmWP0hLAQeEkJ8Uwix/rjPcjhJoDX63AfUHRpASnmLlHK1lHK1qwf8b/O9wyWwcVwxdTZ9Tp5PPHUvBX/iyThXzp+NLgQf+Mmv2N/bP2GYYV5z0SJ6BrLc9Jmf8vz2FuTb3w633BJWT4UI97fcErZdNjVNGEcgzrybiY88/hP2f/51I9Xp4c8fuXreGTm/lJIDe7v48S0P8943/ifbt7Twlncfu1Q6+4b43Hfv444HN3PN2gXjOuYPJV90+bd7/8CT+w5yzcK5Rwx3z/7t/Hz3i1w+ZfaEvz/6whOsv+wSgOmnSNdwnNoumgF7Mi2HRaJrGpdOmcWPdj7PAy1HNqzXLJrD/dv38NUHn8CZYE7MMKXJGBevms2tv93IF3/0EL0D2VDDt9wS9hUcL8Md1AcOhBn8sIEwjPH7YQNw4MDwxR/3qSbU9pev5yN/eqoe2dEpOi7PPLGLf/9/v+Bv3v0tysoTXPP6lcd0rJSSDVsO8P5P30Ym73Dt+oVHDb+7q5cP3fZrTF3n8nmzJgyTc4t87PF7yHsul02g7UAGfH/fXXzsha8cUxrHclx9CEKISwmrwQHwWSnl94/7jGE8XwVuk1I+JYS4HlggpfzXI4WftmSmXPZf17GyfD7vm/UGZpWMLhgtpeQLLzzCN158kumpcj68bD3XzViEdch6/b/ftot/vPP3FFyP1y6dz9tWL2PF1InbWh98Zidf+MFD9A5mmTOtmqvXLuCSVbOYOaVqfPgJOupc0+b3Vcu5svN5YvKV24cw0J9l/+4u9u5sZ8fWVrY830xPVzi8d8WaWfzp+y552XbW/nSOJzfv54Gnd/Lk5n0gBG+9ZiV/8eb1WBP0PXQMDvHLF7Zx64YX6MnmePv5y/nHay/D0Mcb4Oe72/ja5sd5sHUPK6sb+d6Vb6bcHu1nGHKz/LzlAX7Z8iA1dgXfveBTOwCTk9Q1HL+2qxc2yotueRs3TLuK66deTsIYTWd3Psu77/8p2/q7uLZpHn+1dB1LqurHHV/0fD559/388oVt1KVKeMcFK3jD8kXUpJKHnavguPznbY/wywc3o2mCi1bO4orz57J22QzK7rzjuNryJ4sZH7+b/d9+97jO7lOJ7wd0tPazd1cnu7e3sf3FFl7a3ILjuCSSNle/bgVve9/FVFYf3sc1jJSSXc09PPr8Hn7/5Hb2t/VRV5Xi/77nSi5acXgmHwSS5w62ctszm/nt1p2kYjb/cf21XDJ3/PtT8D1+tXcrX9v8OK3ZNH+38lI+tPTCcWF2DTXz7b2/ZMvgHl7dsJ6/nve2U9+pLIR4C/B2IAX8FPgF8Bsp5QXHeqJD4nsXUCul/IIQ4tPADinlT44UfvXq1fJTv/4SP9x3D1k/z6qKBVxbv441VYsxtbCT7bG2/fzLsw+wvb+bqliCP5m5iNdOX8DKmiloUSbeNZThlkef4Y7nt5JzXaaWl3LlgtlcNGcG5zVNITGmk7nguPz28Ze4+9GtbNndDoRjjlcumMrSOQ0smlXPnGk1xH/xs8PWni9c/2aaP/tV6r/xH6QGuumyyngqNZe16V3UuINkzQSarpEoZHGSKTRdw0wPjBtdJCZxlFEhX2RwIEd/b4a+ngy9XelolNFgOMrokGGz1XWlLFo2bWSUUXVt6WFx+kFAc3s/2/d3smV3O5t2trGrOazu1laWcO26hbzximU01ow2jwSBZHtnN4/vOcBDO/by/ME2JLB+9nQ+dOkFrGoaLRgczAxwb/NOfrl3K1v7Oim1bP5iyYW8f+H5WLqOlJLdmYPc1/EUD3RuoBAUqXpO0vfgfu656+4h4GOcpK7h+LW98rxV8m3RCLqEHuPq+gu4qm4tM5PhaJaC7/HNLU/xrW0byLhFllc38IaZi7m2aR4NydH7/NTeZv77kQ08vf8gmhCc1zSFy+fPYv3s6cytqRo3cutgZz93PLCZe5/YTu9gFiFg3vRals+bwlV7NrL42/+J3toSFn5OoER/yjhklBPAl+/becK13iAIyAwVGOjLMtCXpbd7iJ6uNN0dg3S09dPe0k97S9/IsFlN15g1t44lK5pYtXYOK9fMxLIPH4iSzRfZ1dzNS3s7eHF3O8/vaBmZm7R83hSuu3Qx16xdgD2m4z3jFNl4oIVHdx/gwe17aE8PUWJbvHnVUv78otVUJsOmPD8I2Njdwj37t/PrfdsYKBZYVlXPP62+kjV14cJ2jl/kqd4X+V37E2we3EWpkeS9s17P1XVr0TTttBiETwE/lFLuHfPdIinltmM90SHxlQKPAg8ArwbWSikHjxR+xarF8pmNGyj4gnvaHuU37Y/RWxwkocdYU7mY86uWsLJiPqVGkkfa9nHbrhd4sGUPxcCnKpbgooYZrK+fzpq6Jqanysk6Re59aTe/37aLJ/c2U/R9DE1jcUMtK6Y1sKSxnsUNtUyvKkfXNDr7hnhy0z42bjvICzta6e7PRNcBU2rLmTUlnIcwrb6CqbVlTKktp7oiGQ73I8xg9+/uYv+eTpr3hfMQOlr66WwfIJ87vB9C0zVSpXFSpTGSJTESJTaJhE0sYWHHTGIxE9MyME0dw9QxDA1Nm3gegu9LAj/A83xc16fojI7XzueK4VyE4bHaQ+E8BMc5fCKcrmvU1JdR31hOY1MV06ZXMX1WLbPm1VNRVTJyznSmQFv3IK3dgxzsGGB/ex/72/rY19o7MskpbpssmdPAeQunsXbpdObPqEPTBN1DWV7q6GJLWyebWjp4/mAb6UI483ZhfQ1XLZjD65YuYHpVOQNOno1dLTzZ0cyj7ftGJqktrarnzbOXcv3sJdi6xrb0Pjb2beOpnhdpK3RjCoNLalZx/bQr+N6//w+vfeuFXLD4NSMvzcnoOjr++LR93hL5/MbN7Mm0ckfLgzze8wKe9Jkar+PC6qWsrlzMgtQMsp7Lz3e/yO17NrO9PzSkCytqubhhBhfWT2dV7RTKrBh7u/u4+8Xt3Ld9N7u6wtJ+eTzGymmNLJ/awJLGOhY11FCZTBAEkq1723l68wGe3X6QrXs6Rp5RzDJ4e/923vH7H5PIHtvEtpNhOBcqJErZ8OYP0bL+GgxDR9M1dH10ATsIdRYEkiAIRubXjMxDcDycfJFC3iU3PA9hqEAmnWdoqDAyTHosiaRNXUM5DVMrwuGzM2uYMbuWGbNrsWOhAfD8gO7+DG3dg7R0DtDc3s+B9nAeQmvX6OOtr0qxYv4UVi9u4sJlM6guL8HzA/b39rOtvYvNrR280NLOS+1d+FISMwzWzW7i2kXzuGrhHOKmwd50Hxs6D/J4xwEea9vH/2/vTGPkus70/Jxz7lJ7d1UvZDcpkSJFjS2JkuXIiyIJYDKyEcuTieAYWQYYOA4Q+1eCJBMgARIMEiNAEiA/gyRWMpN4BpggDjJjexAhMYyxLNtjjLaxJVkSbIkSKXHtvfa7nZMf91ZvJGW2xGZX098DXFYVUVX363Pfe99zTt3zfavxkNB4fOq2E/zGift56OARVpMOf77yOs8u/5Tnl19lkEXMhi0+O/8Ij889jKcyuulFpkon9s1tp03gU8AzzrmL7/XeYycb7l/90QMcKN3PXOVjzJYf4J2+4weLL/Fny6/QTnooFEer85ycuJO7J+5gvjTHywvLPH3uND+48BaLxcq/qVKFj0zPcXJqjntaB7ij1uL8YpvnzpzjxbPn+On5ywyL20hLnsfxmRZ3zkxxbKbF0akmR1qTlDCcObe8vlL5rXNLvHtpdX1hCuQX0NlmjdlWnZliNedopfJkvUyzUaFRDTFOMegMWVnqsbrcXV/R2WkP6LYH9DpD+r2IfnHxHg4S4ighjtIdr/LUWuH7HkHJK4wloFINKFfCfKFQvUS9WNE50azSnKrRnKpSm6ygA0OnH7PWHbDS7ucrldf6LK5urOa8vNyhv21V9WyrxtH5KY4fnuLE7TMcv32asOrz7mqbM0urnF5c5s2FZd5YWGKplx8jBRybbvHR2+d58Mgh7j40y3LW56fLl3hl6SI/XrzA6Xa+fiHQho/NHubUoWM8Mn87Q9q83n6LV9be5LX2W0Q2xlOG+yZP8Ben7ueByYN0kte4MHie8/1nibJV/s5dP7rRt51et7bvOFlz//abpzhU+QRzlQep+h/ihZV3+eHiT3hl7Q0yZymbkLsbx7hn4jgfahzF2BrfP3+W7507zQsL75JYiwLunJjm/uk5Tk4d5O7mLJOmzMvvXOL5M+f483fO89am389mahVOzE5zbLrFsekWR6YmOTTRYNDJe7tvvLPA6XeXePv8Eg+8+Az/+LlvMBn3b6gxjK483aDK733o1/nu1H3Ew/zCvtPrkucZgtAjCHNtl8sB5Wq+8K1WL1Grl6hPlJmYrDDZrDE5leu7Ui+R4HJddwastgcst3ssFqvwL690ubzUYWGlS7YphY3vGW47MMmxw1McPzzNXUdmuW1+kr5NObO8yttLK7y5sKHtUTK9su9x7/xB/kKh7YNTNd7oLPHq8iVeKlbhr0T56Hu2XOORuaP85UPHuXt6krOD87zWPs3Lq29wpp/PWkz6dT45dZJHpu9jvpRyafgi5/vPcXnwEnV/ns/d8fX9YQg74eBdM+7f/M+/Tau1zGqcD1I8VWK6dDdT4d3EboYzfcfL7Uu83n6bqLj/f9Kvc7x2mDuqh6joBos9y5srXV5ausSba0vrgqz5AXdOTHPnxBRH601KzmPQS1leHXJmMT+wlzrdLTG1KmXmJxvMTdSZa9SZqVcJMWRxRtxP6XciVlZ6LKzkolpa7TG4Ss8b8nQXeVbFPJdRtRxSKeV5jEY5X8LAI/A9As/geQbf03l63yKTqSK/X1yNCh6wOdNp/tw5SLKMJLX5aCFNieJ8W8/3EsX0hwm9QUS3n+cyutYdP8ZoWo0KM80qs606U80qtUaJUtXHCw3WwPJgUOQx6nB+tcOldmdLauFaGHBsusXxmRazzSrVegC+4/ygzZtrS/x8bYmL/c76+2fKVe6bOsiHpyY5WPMxXsTZ/gVOd9/l3GAhz+GE4kh1jpON2zhRC2j5PdaSn3N58PJ6HqOSaRLau3j5TMRvP/af9mxh2vyvTLuvPvWb9HidyOY9zap3gJnSvTSCEyzFFX7WHfLy2lnO9nNvUSgOV2Y5XruN28oHSZKAC92EV5dWeHnp4nraA4D5SoM7J6c4PjHFXKkOMXS7MReXepxeXOb04vKW3Due1hxs1Na1fbBRZ7JUwmRw+1P/h4/+7n+kvnBpXVk7MQgHOKVQzrHanOHpz32RNx799BZt+57BM3p9mnf0SbUh6g1dFzmNHHn67CTNR8JRkhIn+WOu7YTBMKYfJetJ9rr9iO4guuaMWKXkMzVZLTpzNZrNCtVaSFDx0YEmIuNyp8vFtS7n1/JcRtsT38016hyfaXFkepLmZJlSxdAn4e3OSqHtxfWEd1opTkxMc3LqAMebVVoVRc+u8VbvHG903qGd5gvlSjrgw42j3Dsxw5GKwucyi9GrLA5fI3P5aLoZnODyYp3u2jz/6Ff/+a1nCEfurbuv/NFJhpnH0rBKHPmUnOZAHUywtJHNUjeYDI6j9AE6aZkLQ3iz1+d0b4WsSIym0cyVp5krzRCqKnFiWB1aLnQj3lrtcXlbts1WWOZQbYID5RpVFeBZTRo7hoOUdjdipT3g8lrvqqmifa1pVss0K/lWCwJCk2eHNE5BlifyyhJLEudTOkmUT+skccpwmD9GUZqLO8luWO4XrRS+b4qT0VAK89TVo16WH5j1zfgG5SkwCqcdGY7I5qmE1wZDVvoDlvuDLReWEZ7WzNQrTE1UmKiVqFR8/FDjPMeQlKWoz7neGhf6nS3ZNauez7HJBofrJVoVj0pgsSpiMV7i3OAyg2wjlfWBsMFd9TqHS4ZWEOOrFdrxW3TTjVv8PNfi0qpHxzq0nzJR7jNRpMO+0SOEnXDHyZr7l394H6txyNqwgks8JoxiZjIlZSNTb90/TN0/SuKaLMc+ZwcpP+u0WYg3OioVU+JQeZapoIl2ZfqxYqmX8U6nz+m1NoNNCyiNUsxVGxyqNmgFFUp4qFSRRJZeL2a1O2R5bcBCp3dFbQDIkzz+9dd+zN976htMLy9itUZbe83H5eY0f/jpv8UP7n6IOE5yTY+mL5OMOElv2M8VxmhC3yP0TaFpnzAsRhCBhx+aPGuvr/F8nWtbKzLlSLEMs4xuFLPSH7DSH7DaH1yRvRegEvjMjnI4VQPKFQ/tKzJt6bmYi/0O73bXWI23GsXBSo2jExXmaiGTZY3vpQxdjwuDBS4OF7FFV9VTmmPVFserZeZKUPf6ZPYiq/FpYpt3kjQecdRioQtDbSkFMa1yj1BntJOQf3DP07eeIcwcm3N/48nHmG2tMlPuUPU25t0zB92kRJR6kCnKxlItRWi9If5ANwjNASwT9NISy4nm4jDlbH9AJ9soZKNRNIMJ6l4NjxI28xkminaUsTRIuNwb0okdmyuVAZSNT6uU10Uoa79IdZ1f8LPEkSSWKMp74f1BQm8YkyaWX1QfYYSndV4bwTP4Rhf1EfLfDEZ55PPqUBvf5op/LEVOo2LLrCO1ltRa4jQjzjKS98gNP8KNqq2pPJtlpcgRXwoMQWDwAo0qDCNRGUOb0ktjVuPBFScEOEIPDlRDpssBE2VDxQffy8iIGdgey/Hq+kgPHLsNEygAAA9bSURBVKGyzJcDDpcCpkOoewm+6pDaBQbZAusTEE4xiMr00yJfvMmo+BEls/E3JlazOKxxuT3B2uVZfvfz/2HPDGH6zln3hf9+itlGm6mwi7+pYtYg8+gnAVlmMA6qfkopGDIqiabQlL1ZjJomdjU6acBC5Dg3iDg3jLdU7wt1QCuYoKyraBeQZoZeDGvDjMV+zKXekDi7ss/fDMtMBiUqJqCs/DzVtVW4NNd1HGdEUUp/mNAf5mncneO6tR0Yk48MTK5to/KRr1b5KCEv96rYNPDNV6UV44SRrq3NayAkmSXNMpLMEmcZ6XXlb8q17fmaWinXdin0chPx84u8MnmhqISMvk3yOgjD/hW3uitlmQgNs5USrbJPo6Qp+Q6tE2KGdNMOK3F7/aKvcDQ8x+2VEgdDj2ZgqZohmlWi7CKx3TB8m3n0opBBZsBYAi+v8WE2NXMnDVns17i83MQtHuE//8a/vvUMYXuZwa9/70/4zoVvY1oXmJzo0Kp0mQwGBHrrhW2YGeLMwzqFcRCYjMBLtpWVNHh6ElSDxJbpZz6dVLMUZyxGCb3MMLTelpOrpEPKpoyvQpTzsNaQZpph6ujHll6S0Y7SvHpaUWHqahXUIK80VdIegfHwlcYrKk3pYlOo/Px3ilGFG5f/A05tnWvd8nRzuuCi3KBiY0pp0+bUqHhOUTSHomiOs8R2o5LUlT3Fov6ytmjtCAzUQ496YKj4mpKvCAwYk4FKSV3M0A7pZ4P1wiYGS2gSyjplKvCYCgwTPtRMSqCHaLpkbpXMbTWVJDVERQlClMM3GSWTbCmtaR20kzIrwwrL7Tq95WmOupP8k8/+JuVyaUMDY1JCs9Pp8+//7+/xrnmFamuZqUaHZqlH3Rtu+bsyB8PMJ7UGrMIoS8lL8cxW/RtVwugm1tWIbEg381hLYClOWYrTvKqg9bGbytFUTJmSLuGpEKwhs4Y4g2Hi6CWWbpzR3lY9La/vrbbtW1EyPiWzURnQUxqDXtc2gNqka/Kl9fl3ObduLLBNz2zdnVLF9JFi47ZwBShXnDZF4RwsGY7UZSTOEmcpw6IC4HZGFdJGFQDLvqYReNQCQ9nXlDzwjEPrDEtCQsQgG6yn36cotVnSCTXPMhP4NANNw3NUTIKn+kCb1K6u13SB3Oyi1CfODBaF1pbApFs6NACxNaxEFVb6VVZWG7iVeT57+2f49Ycf3fp3/DLWVAZIkoT/8u1v8pP2i9j6ZWoTHSYqfRrhkLo/vMIsIO8pJlbjXC5PrfI6wPoqhcYVHkpVcZRJXUhsPYbW0M+gmzraScbQapKirvLoMXVm3QiMMvjKxygPgwdolNNYp3EuP7GszU/41OZ1CjLrSB1kmSNzjtSy3tPPe0cbPnC1Izk6b7SiqLqWL3rSKq+R7GmFUWC0wuiibrIu3q9H9ZYdrNeZtWQuIXUpidvowRtl8VWWbzp/DHVG3TPUPEXVg7LOSz56KkIT4VwPx5U59fPfOgypy01UFUXRfZ1dUUs5c4puGtKJS6wNKnQ6VeLVFvPcyZcf+TyH569YF3ZlG42JIVyL0++c57/+6f/mkn6TYGKFRr3LRHlAPRhS86ItldYgN8LEmrxer1MoHJ6y+Ca7aqF3TQlUXjc8sQGRNQysppdCN7V0UrteLzwpOkeJ1WRoRgrzlY9XaFtjwGlgs64V1rFeb3mk7VzrjtRujGCzQtd23QyuRG161Ou6Vuu6NkVJTW9d2wovL5mA1i4vv1nUBneFtjMyMpeS2ISsmIbO226rrn2dUdaOhq+peoqKcUUd5QSjYhT9QttXXnMyq0gyg6WopawsnrZbRoYjoszQSUt0ohKrvSq9tTq6e4AHmx/ni489fl1lMn9pDeG9sNbyrR/+kO+9+6e0zUW8+hrlap9aeUgtGFLxYyomvupBgVyYmdN5YXDyHo1WrjiY19N+ARDg8LF42MIoMqdInSaxkDhFaiEuLvqJhdQ5LMUJhS4u/nlRclek3Mtf53vZPgIZJeXLLwJF0fGiPKAmv9Brik05jHL4Ki867muFr8gLkCtXbLl4NRlGpSgSIAFiNteDvRqZVesXKAeg8rwpRtn1WtdXfMYphplPPw3oxiHdYYlev0LcqRIOZ7ir9mG+eOqvMtG4chHWThh3Q3gvFldW+dozT/FG/3Xi8iJhtUu1MqBWHlL1IypeTNkkV5jGiMyqdePFUWgiPy5X6xhtRTPSdq5rD4shtZrUaVKnSKwidZA4SKwrtO7IRkZBvjk3emRd21xD25uTTSpVaJqRtvNPa7Wha43DU4Wmt+k6f7QYlV/4tcrQpChiNrR9bdyma4N1o6mt0flkr+jAjIitYZD59JKQXhTSHZTo96pk3QYT2SF+9cjDfOaTn0DrD5b1YKfafp8pCvcXWmueePRRnuDRa77HOcf3XvgxT7/1LJfSc8T+KqbcIygPKYcxpTCm7CWEXkJoUkKVXqcZgHUx1iWF0PMTzwAGR6hB6c1i5qq9uJuNzbOHbZycxYmKLX6XUNvN5b2/z+i8sHrsNEPrE2Uew8RnmPgMopBoEJIMK+hhgyaz3Dd9L0984mEmrrLiVthgujnJb/21987HtLza5o+f/QGvLL/OKhexpQ5+eUBYiiiHEaUgybVtUkKdYK4xSt6OcxZLhHPRFm17gK8caFBmQ9vXujjeTNYNZ5PxuGLayjnIVD6Bs24qvyBmpXJDiW0+KxClXj4Vl/oMooDhMCQahthBlSBuMhfcxl86/gke+ci915cS/CbzS2EI14NSilMPPsCpB68vRwnA2XcX+O7rL3Bm7R0WowWGuk3m9VF+jA5ifD/BDxJ8k+EXc7y+yYecnrZF79jiKYtWFoPDqAzt9tYU1kdE5PPEmcu3vOeX9wCTzJBmhiTzSFJDnHikiU8W+9gkRCVlAltlUk9xqDbHx47eywN3Hcf3r1zpKewurckGX/j048Dj1/X+KIp48Wdv8NyZV7nYO89qtkJsujh/iA4ivCDG8xN8L9e172W5tgtde2qTvguNa+eAa/eYbwaj6dWRnm0xQs9GIxo70rZHmmmS1Mu3xCONfbIkwCUBJq1Sdg2m/BmON2/n1D0PcOjgzC/c/35ADOEDcPvhGb5w+K/c0O/sdgekScbPz59nbdDh8toK7biDdZal/gqZy+clLZZu1sW6678NtWIqBGqjlkSzPEloAnxluK01j9aKjxw5huf5TDZrN/TvEvYPYRjy0Ml7eOjkPTf0e1dXusRxxEvvvE2WWc6uXCBzGVEWszLYKEIVu4h+NniPb9qKVpq6qaGKH8eNMsxUpkBBq9ygVZukVW1w7OAcYehTrtzIxM23FmIIY0atlic3+1jz5mQpFYSbRd7JqPHYgfeRXVW4Kdz8PM2CIAjCWCKGIAiCIABiCIIgCEKBGIIgCIIAiCEIgiAIBWIIgiAIArAHhqCU8pRSZ5VSTxfbyZsdgyDsBqJtYb+zF+sQ7iMvQv5P92DfgrCbiLaFfc1eTBl9Evg1pdSzSqnfUUrJ4jjhVkG0Lexrdt0QlFJf3TSEfhqYAR5zzn0c8LlGghWl1JeUUs8rpZ5fWFjY7TAFYceItoVbjV3vwTjnvrz5tVIqdM6N6h8+D5y4xueeBJ6EPEXwrgYpCO8D0bZwq7EXU0a/r5S6XyllgCeAn+xBDIKwG4i2hX3NXsxxfgX4A/JiR99yzn1nD2IQhN1AtC3sa266ITjnXiG/G0MQbilE28J+RxamCYIgCIAYgiAIglAghiAIgiAAYgiCIAhCgRiCIAiCAIghCIIgCAViCIIgCAIghiAIgiAUiCEIgiAIgBiCIAiCUCCGIAiCIABiCIIgCEKBGIIgCIIAiCEIgiAIBWIIgiAIAiCGIAiCIBSIIQiCIAiAGIIgCIJQsOuGoJQ6oJT6/qbXvlLqj5VSP1RK/d3d3r8g7BaibeFWY1cNQSnVBL4GVDf9998HXnDOPQx8XilV380YBGE3EG0LtyLeLn9/BvxN4Jub/u8U8M+K588ADwLf3f5BpdSXgC8VLyOl1Cu7F+b7ZhpY3OsgroLEtTN+5X18RrS9N0hcO2NH2r6hhqCU+uq2AP7EOfcVpdTmt1WBc8XzZeDA1b7LOfck8GTxvc875x68kbHeCCSunTHOcV3He0TbY4DEtTOuR9ubuaGG4Jz78nW8rQuUgTWgVrwWhLFGtC38MrAXdxm9ADxSPL8feHsPYhCE3UC0Lexrdvs3hKvxNeAppdSjwN3An13HZ57c3ZDeNxLXzrjV4xJt7z4S187YUVzKObdbgVx7p0rNk/ek/p9zbu2mByAIu4RoW9jP7IkhCIIgCOOHrFQWBEEQgDE3hHFeCaqU8pRSZ5VSTxfbyb2MZ4RS6neUUj9SSv2LvY5lxDi21WZt3WxdjbOui3jG7njB+Gl7jNvpfWt7bA1hH6wEvQ/4H865U8X28h7GAoBS6nOAcc49BBxTSp3Y65gKxqqtrqKtm6arfaBrGLPjBWOr7XFspw+k7bE1BDZWgrY3/d8p4OvF89FK0L3ik8CvKaWeLXoue3HH1nZOsdE+32bjFsi9Ztzaaru2TnHzdDXuuobxO14wntoex3b6QNoeG0NQSn1109DraeAfXuUujetaCXqT4psBHnPOfRzwgcdvVizvwZ61zy/gOcaorZxz7W3a2rV2G3ddXyNG0fb1MVa6hg+u7XFwNGD8V4Juj08pFTrnouLl88A4DGFH7QN5+4yL4b80hm21mV3T1bjrGkTbH4Bx1zXsUFvj0Kg7YZxWgv6+Uup+pZQBngB+soexjBin9tnMOLbVZva63fZ6/9sZx+M1bm0E49lO29lRu43NCOE6eT8rQXeLrwB/ACjgW8657+xhLCO+AXy/WBz1GfI5znFgHNtqM3utq73e/3bG8XiNo7bHsZ22syNt7buFabIS9L0p7jL4FPCMc+7iXsezX9hrXe31/vcDou33x060te8MQRAEQdgd9ttvCIIgCMIuIYYgCIIgAGIIgiAIQoEYgiAIggCIIQiCIAgFYgi3EEqpR5RS/0sppYuskLftdUyCcCMQbd8c5LbTWwyl1H8DBsAZ59y/2+t4BOFGIdrefcQQbjGUUg8CPwJmnHOrex2PINwoRNu7jxjCLYZS6hvAS0DVOfdbex2PINwoRNu7j/yGcAuhlPo8cN4599vAPUqpj+51TIJwIxBt3xxkhCAIgiAAMkIQBEEQCsQQBEEQBEAMQRAEQSgQQxAEQRAAMQRBEAShQAxBEARBAOD/A7py8lVps5t8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import sys, os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import OrderedDict\n",
    "\n",
    "\n",
    "def f(x, y):\n",
    "    return x**2 / 20.0 + y**2\n",
    "\n",
    "\n",
    "def df(x, y):\n",
    "    return x / 10.0, 2.0 * y\n",
    "\n",
    "\n",
    "init_pos = (-7.0, 2.0)\n",
    "params = {}\n",
    "params['x'], params['y'] = init_pos[0], init_pos[1]\n",
    "grads = {}\n",
    "grads['x'], grads['y'] = 0, 0\n",
    "\n",
    "optimizers = OrderedDict()\n",
    "optimizers[\"SGD\"] = SGD(lr=0.95)\n",
    "optimizers[\"Momentum\"] = Momentum(lr=0.1)\n",
    "optimizers[\"AdaGrad\"] = AdaGrad(lr=1.5)\n",
    "optimizers[\"Adam\"] = Adam(lr=0.3)\n",
    "\n",
    "idx = 1\n",
    "\n",
    "for key in optimizers:\n",
    "    optimizer = optimizers[key]\n",
    "    x_history = []\n",
    "    y_history = []\n",
    "    params['x'], params['y'] = init_pos[0], init_pos[1]\n",
    "\n",
    "    for i in range(30):\n",
    "        x_history.append(params['x'])\n",
    "        y_history.append(params['y'])\n",
    "\n",
    "        grads['x'], grads['y'] = df(params['x'], params['y'])\n",
    "        optimizer.update(params, grads)\n",
    "\n",
    "    x = np.arange(-10, 10, 0.01)\n",
    "    y = np.arange(-5, 5, 0.01)\n",
    "\n",
    "    X, Y = np.meshgrid(x, y)\n",
    "    Z = f(X, Y)\n",
    "\n",
    "    # for simple contour line\n",
    "    mask = Z > 7\n",
    "    Z[mask] = 0\n",
    "\n",
    "    # plot\n",
    "    plt.subplot(2, 2, idx)\n",
    "    idx += 1\n",
    "    plt.plot(x_history, y_history, 'o-', color=\"red\")\n",
    "    plt.contour(X, Y, Z)\n",
    "    plt.ylim(-10, 10)\n",
    "    plt.xlim(-10, 10)\n",
    "    plt.plot(0, 0, '+')\n",
    "    #colorbar()\n",
    "    #spring()\n",
    "    plt.title(key)\n",
    "    plt.xlabel(\"x\")\n",
    "    plt.ylabel(\"y\")\n",
    "\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 激活函数与初始化\n",
    "- 梯度消失\\爆炸问题部分时因为选择的激活函数不当\n",
    "- `ReLU` 函数的变体：\n",
    "    - 𝐿𝑒𝑎𝑘𝑦𝑅𝑒𝐿𝑈𝛼(𝑧)=𝑚𝑎𝑥(𝛼𝑧,𝑧)，通常 𝛼=0.01,较小的斜率，保证用不为 0 \n",
    "    - `randomized leaky ReLU (RReLU)`，训练时随机选择 𝛼 参数\n",
    "    - `parametric leaky ReLU (PReLU)`， 𝛼 参数再训练时学习得到\n",
    "    - `exponential linear unit (ELU)`:\n",
    "    $$ELU_\\alpha(z)=\\begin{cases}\\alpha(exp(z)-1)\\quad if\\ z<0\\\\z\\qquad if\\ z\\geq 0\n",
    "\\end{cases}$$\n",
    "- 一般的选择 `ELU > leaky ReLU (and its variants) > ReLU > tanh > logistic`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fb5199844d0>]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD2CAYAAAD24G0VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAYTklEQVR4nO3dd3hUVeLG8e+hBJCOCbj+LBTrLiJiRMFGLyKgWBBpii6irIoVkUUWAUURUGRBEUHsYgGp0lFUUGNZZXcVXcXCqkTpHZLz++MkC4RQZjIz596Z9/M88zCTxMxrMvNyOfece4y1FhERCY9ivgOIiEhkVNwiIiGj4hYRCRkVt4hIyKi4RURCpkS8nyA9Pd1Wr1493k8jIpJUPv7449+stRmFfS7uxV29enWysrLi/TQiIknFGPP9gT6noRIRkZBRcYuIhIyKW0QkZFTcIiIho+IWEQkZFbeISMhEXdzGmGrGmE9jGUZERA6tKEfcjwBlYhVERCRZ7N4Nd94JP/4Yn+8fVXEbY5oAW4BfDvD5nsaYLGNMVnZ2dlHyiYiESm4uXHcdjBgBs2fH5zkiLm5jTBowALjnQF9jrR1vrc201mZmZBS6YlNEJOlYC717w7PPwqBBcMMN8XmeaI647wHGWmvXxzqMiEhYWeuGR554Au6+GwYMiN9zRVPczYDexpglQF1jzITYRhIRCZ+BA2HkSPjLX2DYMDAmfs8V8UWmrLUX5N83xiyx1l4f20giIuEybBgMHgw9esBjj8W3tKGI87ittY1ilENEJJRGj4Z+/aBTJxg/HoolYHWMFuCIiERpwgS49Va45BKYPBmKF0/M86q4RUSi8MIL0LMntGoFL78MJUsm7rlV3CIiEZo6Fbp3hwsvhDfegFKlEvv8Km4RkQjMmQMdO0L9+jB9OpTxsH5cxS0icpgWL4YOHeC009yqyPLl/eRQcYuIHIb334e2baFWLZg7FypV8pdFxS0icggffwytW8PRR8OCBZCe7jePiltE5CBWrIAWLaByZVi4EI46ynciFbeIyAGtXAnNmkHp0rBoERx7rO9ETsRL3kVEUsGqVdC0qbtM65IlULOm70R7qLhFRApYvdqV9pYtrrRPOcV3on2puEVE9vLrr660s7PdmHadOr4T7U/FLSKSZ+1aaN7cbTk2dy6cdZbvRIVTcYuIABs2QMuW7oTkzJlw3nm+Ex2YiltEUt6WLdCmDXz2GUyb5maSBJmKW0RS2vbt0L49LFsGr7ziCjzoVNwikrJ27oTLL3dztCdPdvfDQMUtIilp927o3BlmzYInn4SuXX0nOnxaOSkiKSc31+0P+dprMGqU2xAhTFTcIpJSrIUbb4TnnoOhQ6FPH9+JIqfiFpGUYS3cfrvb1Pfee90tjFTcIpIyBgyARx91R9lDhvhOEz0Vt4ikhAcecEMjPXvCyJFgjO9E0VNxi0jSe/RR6N8funSBcePCXdqg4haRJDd+PNx2G1x2GUyaBMWSoPWS4H9BRKRwzz0HvXq51ZAvvgglkmTliopbRJLSa6/BNddAkybuflqa70Sxo+IWkaQzaxZ06gQNGsCbb7qtx5KJiltEksrChW48u25dV+Bly/pOFHsqbhFJGu++C+3awUknuY0QKlb0nSg+oipuY0wVY0xzY0x6rAOJiETjo4/goovcTuzz50OVKr4TxU/ExW2MqQzMBOoDi40xGTFPJSISgc8/d7vXpKe7oZJq1Xwniq9oJsfUAW631i7PK/F6wNzYxhIROTxffun2iSxb1pX2//2f70TxF/ERt7X27bzSvgB31L0s9rFERA7t22/djuzGuNKuUcN3osSIdozbAB2BdcCuQj7f0xiTZYzJys7OLmJEEZH9/fijK+0dO2DBAndCMlVEVdzW6Q18DrQr5PPjrbWZ1trMjAwNgYtIbP3yiyvttWvd7JHatX0nSqxoTk72NcZ0y3tYCVgf20giIgf2229uTPu//4U5c+DMM30nSrxojrjHA12NMe8AxYF5sY0kIlK49evd7JFvvoEZM6BhQ9+J/Ih4Vom1dh3QPA5ZREQOaPNmN0/7iy/cMvbGjX0n8idJrpUlIsls2za3IvLDD2HKFGjd2nciv1TcIhJoO3a4a48sWQLPPw8dOvhO5J+KW0QCa/dud5W/OXNgwgS4+mrfiYJBF5kSkUDKyYHu3WHqVBg9Gq67znei4FBxi0jg5Oa6nWtefBGGDYObb/adKFhU3CISKNZCnz5uaGTAAOjb13ei4FFxi0hgWAv9+sHjj8Mdd8CgQb4TBZOKW0QCY8gQeOghuPFGGD7cXTxK9qfiFpFAGDEC7rvPnZAcM0alfTAqbhHxbtw4uPNOuPJKN7ZdTM10UPrxiIhXkyfDTTdB27ZugU0JrS45JBW3iHjzyivQo4e72t+UKVCypO9E4aDiFhEvpk+HLl3g3HNh2jQoXdp3ovBQcYtIws2bB1dcAfXqwcyZcMQRvhOFi4pbRBLqnXfgkkvg1FPhrbegQgXficJHxS0iCfPBB9CmDRx/vDvqrlzZd6JwUnGLSEJ89hm0agXVqrkd2atW9Z0ovFTcIhJ3//qXmzlSvrwr7aOP9p0o3FTcIhJX33wDzZq5+dkLF7phEikaTXUXkbj54Qdo2hR27oS334YTT/SdKDmouEUkLn7+2ZX2hg2weDH86U++EyUPFbeIxFx2thse+eUXmD8fzjjDd6LkouIWkZhatw5atIBvv3V7RZ5zju9EyUfFLSIxs2kTtG7tZpFMnw6NGvlOlJxU3CISE1u3wsUXQ1YWvP46tGzpO1HyUnGLSJHt2AGXXgpLl7oNftu3950ouam4RaRIdu1yGyDMmwcTJ8JVV/lOlPy0AEdEopaTA127uvHsMWPg2mt9J0oNKm4RiUpuLlx/vdsM4eGHoXdv34lSh4pbRCJmLdx8MzzzDAwcCHfd5TtRalFxi0hErIW774axY11hDxzoO1HqUXGLSEQGDYJHHnFDIw89BMb4TpR6Ip5VYoypCLwMFAe2AB2ttTtjHUxEgufhh11xX3stjB6t0vYlmiPuzsBIa20L4BegVWwjiUgQjRkDffu66X5PPQXF9O91byI+4rbWjt3rYQawJnZxRCSIJk50JyPbt4dnn4XixX0nSm1R/51pjGkAVLbWLi/kcz2NMVnGmKzs7OwiBRQRv156yU37a9nSTf0rWdJ3IomquI0xVYDHgR6Ffd5aO95am2mtzczIyChKPhHxaOpUt8DmggvgjTegVCnfiQSiKG5jTBrwKtDPWvt97COJSBC89RZ07AhnnQUzZsARR/hOJPmiOeK+DqgH9DfGLDHGdIxxJhHxbMkSd9Go2rXdNbXLl/edSPYWzcnJccC4OGQRkQBYtsxdnrVmTZg7FypV8p1ICtKEHhH5n08+cRsh/OEPsGAB6BRVMKm4RQSAFSvclmMVK8LCha68JZhU3CLCypVuc9+0NFi0CI47znciORhtpCCS4latgqZN3WVaFy+GWrV8J5JDUXGLpLDVq11pb97sZpKceqrvRHI4VNwiKWrNGjc8kp3tTkSefrrvRHK4VNwiKWjtWmjeHL7/3k35q1/fdyKJhIpbJMVs3AitWsGXX8LMmXD++b4TSaRU3CIpZMsWaNMGPv3UXXukeXPfiSQaKm6RFLF9O1xyCbz/vrviX9u2vhNJtFTcIilg50644gp3EvKZZ+DKK30nkqLQAhyRJLd7N3Tp4sazx42D7t19J5KiUnGLJLHcXOjRA159FUaMgF69fCeSWFBxiyQpa+Gmm+C552DwYLj9dt+JJFZU3CJJyFq44w548km45x7o3993IoklFbdIErrvPhg1Cm65BR54AIzxnUhiScUtkmQefBCGDHEb/I4apdJORipukSTy2GNw773QuTM88QQU0zs8KenXKpIknnoK+vSBDh3cXO3ixX0nknhRcYskgeefhxtucNuOvfQSlNDSuqSm4hYJuddfh2uugUaN3P20NN+JJN5U3CIhNns2dOoEZ58N06dDmTK+E0kiqLhFQmrhQjeeXaeOK/By5XwnkkRRcYuE0HvvQbt2cOKJbiOEihV9J5JEUnGLhExWFlx0ERxzDMyfD0ce6TuRJJqKWyREvvgCWraEKlXcUMlRR/lOJD6ouEVC4quv3Oa+Zcq40j7mGN+JxBcVt0gIfPcdNG3q7i9cCDVr+s0jfmmavkjA/fQTNGkC27bB4sVw8sm+E4lvKm6RAPv1V3ekvXatO9KuU8d3IgkCFbdIQP3+uxvT/uknmDcPMjN9J5KgUHGLBNCGDW72yNdfw6xZcO65vhNJkER9ctIYU80YszSWYUQENm9287Q//9xdeyT/pKRIvqiOuI0xlYHJQNnYxhFJbdu2uRWRy5fDlCnQpo3vRBJE0R5x5wAdgY2FfdIY09MYk2WMycrOzo46nEgq2bkTLr8cliyByZPhsst8J5Kgiqq4rbUbrbUbDvL58dbaTGttZkZGRvTpRFLE7t3uKn+zZ7uda7p08Z1IgkwLcEQ8y8lx19N+4w149FHo2dN3Igk6FbeIR9ZCr17wwgtuN/Zbb/WdSMJAxS3iibVuj8gJE6B/f+jXz3ciCYsiFbe1tlGMcoikFGvdbuyjR7vyHjzYdyIJEx1xi3gwdCgMG+Y2+B05EozxnUjCRMUtkmAjR8KAAdC1K4wdq9KWyKm4RRLoiSfgjjvcfO2JE6GY3oESBb1sRBJk8mS48Ua3GvKFF6CErhQkUVJxiyTAlCnQo4e77shrr0Famu9EEmYqbpE4mzEDOneGhg3hzTehdGnfiSTsVNwicTR/vhvPrlvXXZ61rC7LJjGg4haJk3fegfbt4ZRTYO5cqFDBdyJJFipukTj48EO4+GI47jh31F2liu9EkkxU3CIx9o9/uN1r0tNhwQKoWtV3Ikk2Km6RGPr3v6F5cyhXzm3ue8wxvhNJMlJxi8TIf/7jpvsVK+ZKu0YN34kkWWkJgEgM/PCDK+2dO90ONied5DuRJDMVt0gR/fyzK+1162DRIqhd23ciSXYqbpEiyM6GZs1cec+bB2ee6TuRpAIVt0iU1q93s0e+/dbtFdmwoe9EkipU3CJR2LQJWreGFSvcMvbGjX0nklSi4haJ0Nat0LYtfPQRvPqqK3CRRFJxi0Rgxw7o0MEtZ3/+ebj0Ut+JJBWpuEUO065d0LGju+7IhAlw9dW+E0mq0gIckcOQkwPdurnx7NGj4brrfCeSVKbiFjmE3Fz485/h5ZfdBr833+w7kaQ6FbfIQVgLt9wCkybBffdB376+E4mouEUOyFpX1H//u9vg929/851IxFFxixzA/ffD8OFug9/hw8EY34lEHBW3SCEeeMAdYXfvDmPGqLQlWFTcIgUMGQL9+7sNfp9+2l2mVSRI9JIU2cugQTBgAHTtCpMnQ/HivhOJ7E/FLYI7ETlw4J7hkUmTVNoSXCpuSXn5pX3//XDttW54RKUtQaYl75LSrHVDI0OHutWQ48drTFuCT8UtKctadxLywQfh+uvhySdV2hIOUb1MjTFPG2OWGWP+GutAIolgLfTr50r7hhtU2hIuEb9UjTEdgOLW2gZATWPMibGPJRI/+SsiH3rILa4ZO1alLeESzcu1ETAl7/484LyCX2CM6WmMyTLGZGVnZxchnkhsWQt33eVWQvbu7Zazq7QlbKJ5yZYFVufdXwtUK/gF1trx1tpMa21mRkZGUfKJxIy17pojI0a4K/w9/rhWREo4RVPcm4EyeffLRfk9RBLKWrjtNhg1Cm69FR57TKUt4RVN6X7MnuGR04FVMUsjEgfW7inr/PJWaUuYRTMdcBqw1BhzNNAaOCe2kURix1o3LJJ/aVZd5U+SQcRH3NbajbgTlMuBxtbaDbEOJRILubl7TkDmn5BUaUsyiGoBjrV2HXtmlogETm4u3HSTm5/dt6+br63SlmShlZOSdL7/Hq65BpYscYtshg5VaUtyUXFL0rDWXYr1llvc46efdheNUmlLslFxS1JYs8YtXZ82DS64wBV49eq+U4nEh+ZgS+hNnw6nnQazZ8Mjj8DixSptSW464pbQ2rjRzcueOBHq1oWFC6F2bd+pROJPR9wSSm+/DXXqwDPPuEuzfvCBSltSh4pbQmX7drjzTmjcGEqWhHffdZv7pqX5TiaSOBoqkdD49FO3ie8//+kuxzp8OJQt6zuVSOLpiFsCb/duNxe7fn1YuxbmzHHX0FZpS6rSEbcE2tdfQ7dusHw5dOzoCrtKFd+pRPzSEbcEUm4ujBvnZot8+SW89BK8/LJKWwRU3BIw27a564uccoq71sh558GKFXDVVb6TiQSHhkokELKz3TDImDHw22+QmemOsK+8UkvWRQpScYtXK1fCyJFuifr27dC2rZvud/75KmyRA1FxS8JZC++955anT5/u5mB36+ZWQZ56qu90IsGn4paEycmBqVNdYX/wgTvR+Ne/us0Oqu235bSIHIiKW+JuyxaYNMkNiXz3HdSq5Xal6d5dc7FFoqHilrj5+Wd3snHcOFi3Dho0cEfb7dtD8eK+04mEl4pbYmrVKpg1C2bOhEWLYNcuuPRSt1Fvw4a+04kkBxW3FElOjlvVOHOmu61Y4T5+0klud/VeveCEE/xmFEk2Km6J2Pr1MHeuK+o5c+D336FECTeFb+RIaNPGFbeIxIeKWw7LypWuqGfMgKVL3ZH2kUfCRRfBxRdDixZQqZLvlCKpQcUthdq82U3Zyx+v/vpr9/HTToO773ZlffbZOsko4oOKW1i3Dj77DD75ZM/tq6/cQplSpaBJE+jTxw2BHH+877QiouJOMWvW7FvQn3zi5lbnO/ZYqFcPOnWCM8+ECy+EcuX85RWR/am4k5S18NNPbteYvUt69eo9X3PCCXDWWdCzpyvrM86AjAx/mUXk8Ki4Q8patxvMqlXuiDn/lv941Sp30SaAYsXcZVIbN3YFXa+eu851xYoe/wdEJGoq7gDbtGnfMi54f9Omfb++cmWoUQP++Ec3Hl2rlivoOnW0tFwkmai4E2jbNnfd6exsN9Z8qD+3bt33vy9b1hVz9epu7LlGjT2Pa9TQEbRIqlBxR2DHDti4cd/bhg37fyz/tm7dvkW9eXPh3zctDapWdePLVavCySe7P6tVc6WcX8zp6bpGtYhEUdzGmGrAa9ba8+OQ57Dl5sLOnfvfduzY9/62be62deue+4f7eOvWfYt4585D5ypRAipUcEe/FSq4Aq5Va99iLvhn+fIqZBE5fBEVtzGmMjAZiPuI6cSJMHz4vkW89y0np+jPYQwccQSUKbPntvfjKlX2LeGCt8I+Xrq0SlhE4ivSI+4coCPwZhyy7CM93Z1US0vb91aq1P4fO9ht7yIuWNJpaSpZEQmfgxa3MeZJ4OS9PrTIWnu/OUTbGWN6Aj0BjjvuuKiCtWvnbiIisq+DFre19oZovqm1djwwHiAzM9NG8z1ERKRwxXwHEBGRyKi4RURCJqrittY2inEOERE5TDriFhEJGRW3iEjIqLhFREJGxS0iEjLG2vhOszbGZAPfR/mfpwO/xTBOrChXZJQrckHNplyRKUqu4621hW5tEvfiLgpjTJa1NtN3joKUKzLKFbmgZlOuyMQrl4ZKRERCRsUtIhIyQS/u8b4DHIByRUa5IhfUbMoVmbjkCvQYt4iI7C/oR9wiIlKAijtKxpgqxpjmxph031lEUlWqvg8DX9zGmGrGmE9959hb3hZuM4H6wGJjTKFzLRPNGFPRGDPHGDPPGDPVGJPmO1O+vN/jUt858hljnjbGLDPG/NV3loIC+LMK5OsqqO/DfPHsrsAXN/AIUMZ3iALqALdba4cCc4F6nvPk6wyMtNa2AH4BWnnOAyR2r9LDYYzpABS31jYAahpjTvSdKV/QflZ5Avm6Irjvw3xx665AF7cxpgmwBfdiCQxr7dvW2uXGmAtwf9sv850JwFo71lo7P+9hBrDGZ5695O9VutF3kDyNgCl59+cB5/mLsp+g/awC+7oK6vsQ4t9dkW4WHDeF7W8JNAYuBaZ5CZXnANkG495g64BdQcmVtydoA6CytXZ5wHL5iFOYssDqvPtrCdCRmrV2I0CAflb/4/t1VRjjflBe34cF5Q0lDSCO3RWY4i64v6Ux5j5grLV2ve8X8UH23uxtjBkMtANeSWAkoPBcxpgqwOPAZYnOky/avUoTaDN7/glbjoD/yzMIgvC6Kox185m9vg8LcQ9x7q4gv2Cb4X4hS4C6xpgJnvP8jzGmrzGmW97DSsB6n3ny5f1N/yrQz1ob7YW9UsHH7BkeOR1Y5S9K8AX1dRXU9yEJ6K5QLMAxxiwJ0nZpeSeQpgClgBVAbxuAH6Qx5kbgAeAfeR8aZ60NwhEIEJzfozGmArAUWAi0Bs6x1m7wm2pfQflZQXBfV0F9H+4tXr/HUBS3SKzlvembA+9YawN18lvkUFTcIiIhE+QxbhERKYSKW0QkZFTcIiIho+IWEQkZFbeISMj8PykLBDJNwi18AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def elu(z):\n",
    "    result = []\n",
    "    for i in z:\n",
    "        if i < 0:\n",
    "            result.append(np.exp(i) - 1)\n",
    "        if i >= 0:\n",
    "            result.append(i)\n",
    "    return (result)\n",
    "\n",
    "\n",
    "x = np.linspace(-4, 4, 20)\n",
    "plt.plot(x, elu(x), 'b-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "权值衰减`(weight decay)`，一种以减小权重参数的值为目的的进行学习的方法；通过减小权重参数的值来抑制过拟合的发生\n",
    "- 一开始将初始值设为较小的值，当不能将权重初始值设置成相同的值；误差反向传播算法中，所有权重都会进行同样的更新，将无法正确进行学习"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEFCAYAAAAPCDf9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAQv0lEQVR4nO3df5DcdX3H8edLIhKCVSjXKGqaUpGpVmM0UmgjPRywpoq1aIutP8Zfk9ai045ORyq0UltbWjva1qozmSKg/WXE8TcWtMWScUBMxlGx4qA1qLFAFAyD9cco7/6x30yO2+N27za3e3ef52PmJt99f3/s5/O+vdd+77u7uVQVkqQ23G/SA5AkjY+hL0kNMfQlqSGGviQ1xNCXpIYY+pLUkFUR+kmel+TSIbd9UZLLlnhIE5fk/CQHknwnyauH2P6iJBeNYWgTk+RJST6Q5Kok24bYftX35KAkLx7m56KFniT5YpJbu69vDLH9iurJmkkPYFRJngG8FXj/pMeyXCQ5DfgtYDNwFHB9kp1V9fXJjmxykjwI+Gfgd4ACdiY5uarunOzIJi/JFPBG4MOTHsukJTkaqKp6yKTHslRWfOgDLwb+DHjspAeyjPwf8NtV9T8ASfYCJwDNhj7wUODCqroGIMnXgYcDzYc+8GbgY5MexDKxCfj8pAexlFbD5Z3nAN8e5QBJnpXky0m+meQtXe1RSb46Y5vfT/LWbvmUJJ9JcnuSHUnS1S9K8vokf53k20keMMq4FquqPltVX+jGdALwCODGhR4nye8m+VqSfUle09V+Jck1M7Z5c5I/7JZ/NclNSW6b+etuksuSvDzJO5LcPOL0FqWqbqqqnUmOSPJs4P7Afy/0OKupJ904zqL35LdjhGOspp5sBrZ2WfCVJM9czEGWdU+qasV/AS8CLuuW/wG4ddbXu+badkbto/S+2WvoPcs/uqt/Hnh8t/wfwDRwJHAzvTOCo+idIf16t81FwD7gAmBq0n3pxnQpcDHwgTn68hcztrsIuGjG7aOAT9L7DeFoYD9wTNej24Gf7La7GfgZYAr4CrABeCDwBWBzt81lwNeAlwLHTbgfrwK+B5zXek+Atd1j/KTusX2ZPeGFwPO75VOB2+jlw6rpyWq4vHMvVfUK4BUL3O0lwG8Ar6X3A/BT9M4C3ws8qzvjfxRwLfAYYCNwVbfvkcCjgfd1tz9fVW8YYQqHTfdi5VZ6T1zfXci+VfX9JC8EXgA8GTgOOL6q9ia5Ejg7yQ3Ad6rqq0nOBh4G3NAd4gH0evWZ7vaVVXXJ6LMaTVW9KclO4FPAL1fVlxew72rryevonRDdnORhAFX1aws5wGrrSVW9c8by9UluAV5XVTfMs9vsYyzrnqy60F+oJA8GdtN7IesN9J51D7oC+CfgJuBDVXVPdynny1X1c93+RwFHzNjn+rEMfIAkjwD+kd5vIQsK/G7/n6X3JHcR8Grg52esvgJ4GfAQ4D0HdwGuqapt3f7rgHtm7DPRviR5JHB0VX2uqr6R5NPAycDQob/aegKcDRyf5FX0Tl7WJvmJqjpn2AOstp50Yf1vVfXDrvQw7j2+YY6xrHuyGq7pj+qRwI+At9H7dfeJB1dU1Y30flV7JYe+QTcBRyd5cpL7Ae+i901cNrrXEj4IvHEhZyizbAb2Au+gF44Pn7HuY8AvAL9J70EMvQfm5iQnJzmy2+api7zvpXAC8O4kD07yUOBJHDqTGtaq6klVPaaq1lfvnSrnAO9eSOB3VlVPgNOBVyc5Osl5wA9Z+Au7y7onhj58tvv6X+BP6H2DHzVj/XvpPTFcC9CdAZwLvIXe9fvvA28f43iH8TTg8cD5OfR+42cv8Bgf7/69DXgucPASF1X1A+A/gXuqe4dQVd1O78nvg/Qe8NdV1QdGncjhUlXXAu8EvgjsAv6gqr65wMOsqp4cJqutJ6+l9/NzG71gfmY3j4VY1j1J92KB5pBkDb2A31pVL5/0eJaLri+vAX5UVX816fEsB/aknz3ptxx60vw1/QGuoPcunadNeiDLzA30rgFPT3gcy4k96WdP+k28J57pS1JDvKYvSQ0x9CWpIcv6mv7xxx9fGzdunPQwltyePXu+VVVTg7fsaaEv9mRuC+mLPelnT5Z56G/cuJHdu3dPehhLrvvU39Ba6Is9mdtC+mJP+tkTL+9IUlMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDVkzobzz/I2w8/yOTHsayYk/62ZO52Zd+rfZkxYS+JGl084Z+kjVJvpbkE93XY5P8aZJPJ3nrjO0WXZMkjc+gM/3HAf9aVdNVNU3vL75sBU4Bbk9yZpInLra2NFOSJN2XQf/L5qnAM5KcQe8Phn8JeG9VVZKrgG3AgRFqH599h0m2A9sBNmzYcFgmKUnqGXSm/2ngzKo6Bbg/sBbY1627A1gPrBuh1qeqdlTVlqraMjU19H+nLkkawqAz/c9V1Q+65d0cCn6AY+g9adw9Qk2SNEaDgvddSTYlOQJ4Fr2z9a3duk3AXmDPCDVJ0hgNOtN/PfAvQIAPAn8O7Eryd8DTuq9bgL9cZE2SNEbznulX1Y1V9biqemxVXVBV9wBnAruAbVX11VFqSzkxSVK/Bf+N3Kr6HnDF4apJksbHF1MlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWrIUKGfZH2Sz3TLlyS5LsmFM9YvuiZJGp9hz/T/Blib5BzgiKo6DTgxyUmj1JZiQpKk+7Zm0AZJngJ8F7gVmAZ2dquuBrYCm0eo3TzqBCRJw5v3TD/JkcAfA+d3pXXAvm75DmD9iLW57nN7kt1Jdu/fv3+h85EkzWPQ5Z3zgbdV1Xe623cDa7vlY7r9R6n1qaodVbWlqrZMTU0tbDaSpHkNCv0zgfOSfAJ4PHA2vcsyAJuAvcCeEWqSpDGa95p+VZ1+cLkL/mcCu5KcAGwDTgVqhJokaYyGfp9+VU1X1V30Xsy9Hjijqg6MUjucE5EkDTbw3TuzVdWdHHoXzsg1SdL4+IlcSWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaMlToJzkuyVlJjl/qAUmSls7A0E9yLPBh4BTgmiRTSS5Jcl2SC2dst+iaJGk8hjnTfxzwqqp6A3AV8BTgiKo6DTgxyUlJzllsbWmmJUmay5pBG1TVfwEkOZ3e2f5xwM5u9dXAVmDzCLWbZ95fku3AdoANGzYsYkqSpPsy7DX9AOcCdwIF7OtW3QGsB9aNULuXqtpRVVuqasvU1NRC5yNJmsdQoV895wGfA34RWNutOqY7xt0j1CRJYzLMC7mvSfLC7uaDgYvpXZYB2ATsBfaMUJMkjcnAa/rADmBnkpcBNwLvB65NcgKwDTiV3iWfXYusSZLGZOCZflXdWVVnVdXpVfV7VXUAmAauB86oqgNVdddia0sxKUnS3IY50+9TVXdy6F04I9ckSePhC6mS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0JekhgwM/SQPSvLRJFcneV+SI5NckuS6JBfO2G7RNUnSeAxzpv884E1V9VTgVuC5wBFVdRpwYpKTkpyz2NrSTEuSNJc1gzaoqrfNuDkFPB/42+721cBWYDOwc5G1m2feX5LtwHaADRs2LGgykqT5DX1NP8lpwLHA14F9XfkOYD2wboTavVTVjqraUlVbpqamFjQZSdL8hgr9JMcBbwFeAtwNrO1WHdMdY5SaJGlMhnkh90jgPcAfVdUtwB56l2UANgF7R6xJksZk4DV94KXAE4ALklwAXAq8IMkJwDbgVKCAXYusSZLGZOCZflW9vaqOrarp7utyYBq4Hjijqg5U1V2LrS3FpCRJcxvmTL9PVd3JoXfhjFyTJI2HL6RKUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktSQoUI/yfoku7rl+yf5UJJPJnnJqDVJ0vgMDP0kxwKXA+u60iuBPVX1S8BzkjxwxJokaUyGOdP/MXAucFd3exrY2S1fC2wZsXYvSbYn2Z1k9/79+4eeiCRpsIGhX1V3VdWBGaV1wL5u+Q5g/Yi12fe3o6q2VNWWqamphc1GkjSvxbyQezewtls+pjvGKDVJ0pgsJnT3AFu75U3A3hFrkqQxWbOIfS4HrkzyZODRwKfoXbJZbE2SNCZDn+lX1XT37y3AWcAngTOr6sej1A7rbCRJ81rMmT5V9U0OvQtn5JokaTx8IVWSGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSFrJj0ASeOz8fyPsPfip/fVAPZe/PR7Lc+1fq7bK918cz5oZm9mW2l9WHGhf1+NH6fZD/6Ztbks9Q/JQnsy1/hnr1vplsPjZD6TDND76s3M+lzbzK4tVY8n1Zv55rPYdYfTQnPnvqy40F8OhvmBGLTfJMN1OTyA78tqedIZZCEB2kpPDhqmN631BIbLnWH6MpFr+kkuSXJdkgsncf+S1Kqxh36Sc4Ajquo04MQkJ417DJLUqlTVeO8w+Xvg36vqyiTPBdZW1aUz1m8Htnc3Twa+1C0fD3xrrINdWjPn89NVNTXsjkn2A7ew+noCh+ZkTw5Z1GNlRk9mH2M1sCdzG/jzM4lr+uuAfd3yHcATZq6sqh3Ajtk7JdldVVuWfnjjMcp8Dn4zV1tPYPFzsif9Zv7Qr7a+2JO5DTOnSVzTvxtY2y0fM6ExSFKTJhG4e4Ct3fImYO8ExiBJTZrE5Z33A7uSnABsA04dcr++Sz4r3OGYz2rrCYw+J3uydMdYTuzJ3AbOaewv5AIkORY4C7i2qm4d+wAkqVETCX1J0mT4IqokNWRFhP5q/ARvkvVJdo2wvz3p39+ezH0M+9K/f7M9Wfahvxo/wdu9pnE5vc8sLGZ/e9K/vz2Z+xj2pX//pnuy7EMfmAZ2dstXc+jtnivZj4FzgbsWuf809mS2aezJXKaxL7NN03BPVkLoz/4E7/oJjuWwqKq7qurACIewJ/3sydzsS7+me7ISQt9P8PazJ/3sydzsS7+me7ISJusnePvZk372ZG72pV/TPVkJf0RlsZ/gXc3sST97Mjf70q/pnqyID2f5Cd5+9qSfPZmbfenXck9WROhLkg6PlXBNX5J0mBj6ktQQQ1+SGmLoS1JDDH1Jasj/A1oC/Ju0W5h3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "\n",
    "def ReLU(x):\n",
    "    return np.maximum(0, x)\n",
    "\n",
    "\n",
    "def tanh(x):\n",
    "    return np.tanh(x)\n",
    "\n",
    "\n",
    "input_data = np.random.randn(1000, 100)  # 1000个数据\n",
    "node_num = 100  # 各隐藏层的节点（神经元）数\n",
    "hidden_layer_size = 5  # 隐藏层有5层\n",
    "activations = {}  # 激活值的结果保存在这里\n",
    "\n",
    "x = input_data\n",
    "\n",
    "for i in range(hidden_layer_size):\n",
    "    if i != 0:\n",
    "        x = activations[i - 1]\n",
    "\n",
    "    # 改变初始值进行实验！\n",
    "#     w = np.random.randn(node_num, node_num) * 1\n",
    "#     w = np.random.randn(node_num, node_num) * 0.01\n",
    "#     w = np.random.randn(node_num, node_num) * np.sqrt(1.0 / node_num)\n",
    "    w = np.random.randn(node_num, node_num) * np.sqrt(2.0 / node_num)\n",
    "\n",
    "    a = np.dot(x, w)\n",
    "\n",
    "    # 将激活函数的种类也改变，来进行实验！\n",
    "    # z = sigmoid(a)\n",
    "    z = ReLU(a)\n",
    "    # z = tanh(a)\n",
    "\n",
    "    activations[i] = z\n",
    "\n",
    "# 绘制直方图\n",
    "for i, a in activations.items():\n",
    "    plt.subplot(1, len(activations), i + 1)\n",
    "    plt.title(str(i + 1) + \"-layer\")\n",
    "    if i != 0: plt.yticks([], [])\n",
    "    # plt.xlim(0.1, 1)\n",
    "    # plt.ylim(0, 7000)\n",
    "    plt.hist(a.flatten(), 30, range=(0, 1))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 使用标准差为 1 的高斯分布作为权重初始值时，各层的激活值呈偏向 0 和 1 的分布；偏向 0 和 1 的数据分布会造成反向传播中梯度的值不断变小，最后消失；随着网络越深，梯度消失`(gradient vanishing)`的问题会更加严重\n",
    "- 使用使用标准差为 0.01 的高斯分布作为权重初始值时，各层的激活值集中偏向 0.5；此时不会出现梯度消失的问题，但多个神经元输出几乎相同的值，神经网络会出现“表现力受限”的问题\n",
    "\n",
    "   \n",
    "- 直观来讲，每一层输出的差别变化幅度应当与输入变化幅度相当；梯度在反向传播经过每一层前后，变化幅度相同\n",
    "- `Xavier` 初始化，使用 `sigmoid` 或 `tanh` 激活函数时的**最佳实践**\n",
    "    - 以 0 为均值，$\\sigma$ 为标准差的正态分布：$\\sigma=\\sqrt{\\frac{2}{n_{inputs}+n_{outputs}}}$\n",
    "    - $-r$到$+r$之间的均匀分布：$r=\\sqrt{\\frac{6}{n_{inputs}+n_{outputs}}}$\n",
    "    - 简化为 $\\sqrt{\\frac{1}{n_{inputs}}}$\n",
    "    - 此时各层激活值基本避免了上述两个问题\n",
    "    \n",
    "        - `He` 初始化，使用 `ReLU` 激活函数时的**最佳实践**，初始化为以 $\\sqrt{\\frac{1}{n}}$ 标准差的高斯分布；相比 `Xavier`，`ReLU` 负值区域的值为 0 ，为了使它更有广度，需要 2 倍的系数；"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-04T06:54:35.696650Z",
     "start_time": "2020-04-04T06:54:35.102169Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 50000)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABDAAAAFOCAYAAABwhO0AAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeZwcdZn48c+TcAYIZwQkIKKcCrKKIgpLuFRAFkEQEWRx3Y2yconiCiIGvEBFQcHFLCrCKocnotywCAiIYVf8cQSQ+5ZASISEw+T5/VHVk0qn5+ienumemc/79arXdFV961tPdc/UdD/9PSIzkSRJkiRJ6mbjOh2AJEmSJElSf0xgSJIkSZKkrmcCQ5IkSZIkdT0TGJIkSZIkqeuZwJAkSZIkSV3PBIYkSZIkSep6JjAkSaNSRKwfEVkuZ7epzgfL+h5sR339nKsW+7W97J9WKTNlqOMZDkPxmnWD0XpdkiQNt6U6HYAkaeyIiKw9zszoZCwAEXEwsD5AZk7rZCySJEnqmwkMSdJYdjCwffl4WufCkCRJUn9MYEiSRqXMfBBoayuPzFy/nfUNRtliZFqHw5AkSRo2joEhSZIkSZK6ngkMSZIkSZLU9UxgSJK6RkRMqczWMK3ctl5EnBIRMyPihYh4LiJujIh/j4heu0L2NfNDRFxbDii6fWVbNlim1R3X7ywkZbyHRsRPI+LuiHg+Il6OiL+W5/2PiFi5pSdo8fP0OgtJ3fM40GVa4zP1XNOXI+KWiHi6vJ4nI+LKiDgkIpYZYMybR8T3I+KhiHgxIp6IiCsiYv/BPRs99a9expYRcesAj/mnynNwSoP9m0TE0RHx64i4PyLmRcRLZeyXlb+Hy7Uh9j5nnRlE2W0i4j8j4s7yb+fFiHg4Ii6IiN0HcPz4iPhwRFwcEY+Ux88vH/9vREyPiL0jYsLArlSSpNY5BoYkqWtFxHuA84BV6nZtUy7vi4g9MvOlYQ+ugTKRcA2Nx96YRJEw2R74VETsnZk3DGN4LYmIY4AvAMvW7VqzXHYGPhkR783Me/qo5xDgNGDpyua1ymWXiNgX+OxgYs3MZyLiUuCfgDdHxKaZeVc/hx1YeXxuXcwHAT/q5bha7O8Gjip/D/s717CJiBWAs4APNti9brl8ICJ+C+yfmX9rUMcawCXAWxvUMblc/gH4N2Av4FftiV6SpMZMYEiSutWWwNEUyYDvATcBLwFbAR8HVgB2AT4HHN9k3ccBawBfAt5QbturQbmZTda7XBnvHcD/AHcBz5Tb1wXeB7yFIpnxm4jYshxstN1up/H1VAXwdeB15fpzSxSI+BZwZLn6N+B84BZgDsWH9/cBOwIbAteV1/Nkg3o+AHy3sulS4KLynJsAHx1AvAN1LkUCA4rkxOd6KxgRE4E9ytXbM/NPdUUmAAncClwH3A3MBiYCrwH2AzaieA4vLa9/iedxuEXEssBVwNvLTQ9TJALvoPgbej1wELAxsDvwq4jYJTMX1lX1XyxKXvylrOMeYD7Fc7Ax8I/A1kN2MZIkVZjAkCR1qz0pPnjtnJn3VrafHxEXAr+n+D92aER8uZlWGLWWDxFxZGVbO749vgvYIjP/Xy/7v1R2l/hvYGWKlg0facN5F5OZs+jn2/CI+CKLkhfXAmfU7d+TRcmLG4H3N0hOfCciplIkmNYETqXuG/+IWKVSdwJTM/OsujLfBH5Le5IYF1MkWFYGDoiI4zIzeym7D0VyCYrXpN71wOsy84FGB5fdbo6iSAS9BjgCOKH10NvmJBYlL84EjsjMl6sFIuLrFC00DqJIQk0ty9b2v4pFiaAZwJTMfKHRySLiNW2NXpKkXjgGhiSpmx1Yl7wAIDNvAS4oV1cF3jasUfUiMx/qI3lRK3Meiz4s7xcRS/dVfihExIcoWqFA8c36+zPzlbpiJ5Y/ZwF7NGpZAZCZ01nU9WKfiFi3rsjBFK1dAH5Un7wo6/gbRWuGJboxNKtMZP20XH0NsG0fxWvdRxYCP25Q1x29JS/K/Qsz8xsUrTMAPtx8xO0VEWsD/16uXp2Zh9QnLwDK1/tfgfvLTUfVFdmARe8Tf9Jb8qKs66HMfGhwkUuS1D8TGJKkbvV/mXl9H/uvqTzebKiDabMby5/LA1sM54kj4u3A98vV5yiSE8/WlXlTJa4f1O9voJaQGQ/sVLev2qpiiUEyazLzCRq3gmhFdSyLAxsViIh1WDSI67WZ+eggzld7PV9XjhvRSR8AaoOq9vp8Q08So5YI3DAi1q/snld5/AYkSeoCdiGRJHWrm/vZ/1jl8apDGUizImJrig/Ob6f4JnslFh+8smoyxRgLwxHXehRdS5YD/g58IDMbjfOxXeXxuIh4Xz9Vr1N5vGnlfEEx5gfAXzPz9n7quRo4pJ8yA3E98BBFC4x9I+KwBq0QDmDRFzl9Jk4iYmeKrjFvBdajeD3H91J8HYpWK51Sfe1eNYDXrvq3synwYPn4DuBx4NXAR8vX8r+AWxqMlSFJ0rAwgSFJ6lb9fQisjnkx6Gks26GcTvQsmutKMHGIwllMRKwI/JpirAqAIzPzyl6Kr195/OlyGajqB+KVKQZbhaKrSn8GUqZfmZkR8WPg2DKe3YFf1hWrtcyYD/y8UT1RTHd7IfCuJk4/LK9nH9avPD67yWN7XrvMXBARH6N4bpYB/qVcnouIm4AbgMszc1iSb5IkgQkMSVL3Gonf8p7BouTFSxRTUP6RorXIC8CCct+OwGHl496+yW+biBhHMcbDm8pN383MM/o4ZOVBnG6ZyuMVK4/n1RdsoNdxFlpwLkUCA4pkRU8CIyI2BzYvVy/KzLm91PEzimlioRif42LgT8ATFNdT+x39IMUYHjAMr2c/2vXakZm/iYi3AdMokkBLU0xpvGu5fDkibgeOzszLBnFeSZIGxASGJEltUI4f8NFy9VFg+8y8v5ey6zTaPoROYtGMEldRzJbRl+crj6dk5u9aPG+1ngkDKL9C/0UGJjNnRsQMiml3d4+IVSpTnFZbyJy75NEQEf/IouTFbcAumfl0L2Xf2aaw+1Umo/pSe87/DiyfmX8fzPky8zZgr4hYCXgn8A6KqVPfQZHQeCNwSUR8ODOXGAhVkqR2chBPSZLaY0cgyscn9Za8KA3btJMRcTBwdLl6N7DvAD7UVscXGcwAjnNY1Kri9QMoP5AyzaiNbbEssC/0JAD2L7c/DVzRy7E7Vx5/rrfkRakdr2dtjI5l+iy1aEaX3tReu6WAjQYVUUVm/i0zL8vM4zNzCrA28K1ydwDfjIhOtz6RJI1yJjAkSWNZTzeVcpDCwViz8vi+fsq+e5DnGpCI2A74Xrn6LPDeSiuEvlRbXOzVa6l+ZGYCM8rVV0VEf8mQ+hlMBus8ipYIsGjMiykUA6cCnNdHMmdAr2c57smU1kPsUXtdXt1Pua372d+W164/mflMZh5F5fUFNhyq80mSBCYwJEljW7WLw2C7L1THeHhdb4UiYk+GYerUiNgA+AXFN/qvAPtk5kAHyZxBMQsFwM4RscsgQqkOnnlUb4UiYk2KmUHaJjP/yqIWFtuVs7BUp1Vt2H2kNKDXk2LWlEmtRbiYO8ufrylfu94c3k8957OoNccnI2KtQUfWtwcrj+2aLEkaUiYwJElj2QOVx28eZF1/rDz+dEQsMbVrOb3qDwZ5nn5FxESKASdr3Q0Ozcz/GejxZcuJYyqbLoiIPluNRMSmEfGfDXb9iEUzynyk7NJSf+yKFB+8h2IGj1o3kqAYo+T95frdmTmj8SHA4q/n8RGxbH2BiNiDYnyRdqgOgnlyoxZBEXEii3dtWUJmPgJ8p1xdHbg8InrtmhOFnSLic3Xb3x0RR5QzsfR27OuBWnLrefpveSRJ0qCYKZckjWVXs+gb7e9HxLeAh1g0W8hfmmi1cBNwK/AWiqksZ0bEmRTjTixPMUbGfhQfpH8CfKgdF9CLLwKblY//CPw1It7XzzEzM3NmbSUzLy4/MB9PMb3mZRFxPXApxXP0d2A1ijEytqeY1WMBRYsEKvU8FxGfAC6guPYfRsQ+FFO6zgE2pkgsrEfRWqPd3R5+RTGDyErAZ1k0xkRfrS8oY3kMWAd4G3BnRHwfuJ9iJo7dgD0oWmr8Ath7kHH+APgMxXO6D3B9ORXsLIrn5oMUA5KeXz7uyzHAlhRdcrYoY78IuA54kmLwzTUpZqXZhaLbytXAlyt1rA2cCnwtIv4H+APFtc+jSIy9FfgAi1ounZqZ81u8dkmSBsQEhiRpLPstcAOwLcUAkvVTi55AMYVkvzIzI+KDwDXAuhRjAhxfV+wl4N8pxt4YygRG9Vvzt7J4N47eLHGtmfmFiHgEOIWidcR25dKbRxttzMwLI2IN4DSK9x67l0vVzyk+eLc1gZGZ8yPi58DBLEpeJMW0sv0dtw/FVLirAhuw+Ad8KMatOIAiwTGoBEZmPh0RB1IkQ5ajmPGjfnaT31Ake/pMYGTmKxGxG8XrdghFwmKfcunNY3XrtfFhlqEYs6W3FjgJfBv4Ql8xSZLUDnYhkSSNWZm5gOIb6M9StKCYzaLWF63U9xfgH4CvAncBL1I0rb8bOB14c2YOeReSdsrMsyhm2fgUxXgSj1MkYl6i+Db/OuDrFN/29zp2Q2Z+l6KbztnAIxTjNDxFMa3rhzJzH4qxOoZCfWuLGzLzwf4OysybKVopnE7RPeJlilYjtwMnA2/KzEvaFWRmXkrRcuKHwMPl+Z4G/odi6td/ysx5vdewWF0vZ+ZhwCYU3Vz+UNb1d4pWFA9QJGeOBbbIzH+uq+Jcims/CrgI+AvFjDILKJ6DP1E8L2/JzCMzcyGSJA2xKLq5SpIkSZIkdS9bYEiSJEmSpK5nAkOSJEmSJHU9ExiSJEmSJKnrmcCQJEmSJEldzwSGJEmSJEnqeiYwJEmSJElS1zOBIUmSJEmSup4JDEmSJEmS1PVMYEiSJEmSpK5nAkOSJEmSJHU9ExiSJEmSJKnrmcCQJEmSJEldzwSGJEmSJEnqeiYwJEmSJElS1zOBIUmSJEmSup4JDEmSJEmS1PVMYEiSJEmSpK5nAkOSJEmSJHU9ExiSJEmSJKnrmcCQJEmSJEldzwSGJEmSJEnqeiYwJEmSJElS1zOBIUmSJEmSup4JDEmSJEmS1PVMYEiSJEmSpK5nAkOSJEmSJHU9ExiSJEmSJKnrmcCQJEmSJEldzwSGJEmSJEnqeiYwJEmSJElS1zOBIWnAImJaRGRETOl0LJI0WBGxYnlP+02nYxmJIuIb5fO3VadjkdS/iLg2IrLB9qUj4oSIuDciXir/rt/XiRil/pjAkPpR3sQzIh6KiOV6KfNgWWap4Y5P0shTua8MdDm40zEPl27+UFxJeGRE3N3bPT8iZkXEi8Mdn6SRJyKmlPeUa/sos35Z5sEhCuNTwPHA48A3gBOAmUN0LmlQ/LAlDdx6wJHASZ0ORNKId0KDbUcCKwOnAc/V7fvTkEekZm0EfBw4vdOBSNIAHQRMaLD9vcDzwC6Z+fLwhiQ1xwSGNDCzgQSOiYizMnNWpwOSNHJl5rT6bWUri5WBUzPzwWEOSc15iuJDwBci4pzMnNvpgCSpP5n5cC+7Xg08Y/JCI4FdSKSBmQd8EZgIfKGZAyPiAxFxXUTMiYj5EfH/IuKYiFi2QdkHy2ViRHyzfPxKREwr9/eMQRER+0fErRExLyIeL8svW5bbseznODciZkfEuRGxeoPz7RAR0yPizrLs/Ii4PSK+0Ft3GUmdExEzIuL5iFg+Ir4UEX+JiJcj4vRyf6/dLyLijeW+0+u2/6zcPikijijvBy9GxBMRcXpErNBLLOtHxHcj4r6y/KyIuDkijq4r966I+EFEzIyIv5X3rD+X98Gl68rOomjKDPDHSneN5+vKrRgRx5f303llvddHxN69xLpcRHyxvKe+VMZ8PLB0o/ID8BzwNWAN4JhmDoyID0fE78t77ryIuC0iPlX/XJRlZ5X35NUi4jsR8XBE/D0iPl3u73m9I+LgiPhTeR9/NCK+UuviEhG7RsQN5e/OM+XrMbHB+Qb8WknqrLr3hPtExC3l3+yzEXF+RKzT4JjFxsCIiLPL9dcCr6nccx+sO65r38tq7LEFhjRwZwCHAh+LiO9k5j39HRARX6F4czsL+AlF87xdga8A746IXTLzlbrDlgGuAVYDrgDmAg/UlTmsrOdXwLXAu4BPAqtFxEXA+cBvgenAO4ADKd5o71pXz38AmwA3luWXA94JTAOmRMTOmbmgv+uUNKzGAb8BNgYuB54BHmpDvWcAO1HcCy4DdgE+AbwG2KNaMCK2LctNBK4GfgqsBLwB+Dzw9Urx44G1gD8AFwErAttR3Ae3jYj3ZmbtDfXXgPcB2wD/RdEfG6DnW8GImERx39sMuKUstwzF/e3nEXFMZp5UKT8O+HV5PXcD36ZoPXEo8JZmn6SKUyi6kBwZEd/NzEf6OyAivk1x/34KOAd4keK5/QawU0Ts0eCeOwG4rrzGSygS6vXnOobi/8BFFP8/di23TYyIm4GzgIuBm4HtgY9QvF771tXTzGslqTv8O/BPFPe53wFbA/sBb4qILTPzpT6O/RXwIEUXRoBTy5893RhHwHtZjTWZ6eLi0sdC0XXk0fLxPuX6L+rKPFhuX6qybZty28PAWpXtS1G8kUzg2F7quQpYoUEs08r9c4BNK9uXBe4AFlB8mNm+sm8ccGV53JZ19W0ARIPzfLEsv18v55/S6dfFxWW0LZW///X7KDOjLHMLsEqD/d8o92/VYN8by32n123/Wbn9HmDtyvZlKufbrLJ9AvAEsBDYs8F5Jtetb9DLtXyrrHv3gV5DXbyfqNs+geLN+9+BjSrbp5blrwaWrmxfE3i03PebAb5GK5blZ5brB5fr59SVmwW8WLdtl7LsvcDqdc/zVeW+wxvUkxQfTJbr4/V+Gnhd3XPxAPBKWcfbKvvGA78vX78Nh/K1cnFx6X8BppR/R9f2UWb9ssyDlW3Tym1zgc3ryv+k3PeBuu3XAtmg/gerdVe2d/17WZext9iFRGpCZv4MuAnYq/wGsi//Uv78UmY+Wanj7xRNpBcC/9rLsZ/KzBf6qPvbmXlXpc6XgAsobvC/zczfVfYtBP67XH1T3fXcn5mNvk2rZeDf3UcMkjrnmMysH+hzsI7PzCdqK1n0hf5Rufq2Srl9Kb6lPy8zL6qvJDMfrVu/v5fzfav8OeD7TERMBvameKN/Rt155gHHUnxA/2Bl10fKn5/NyreEmfkUgx+U+RzgNuDAiHhzP2Vr/xO+kJnPVOJ4mUXdZnr7n/DJzOxrVpNvZOZ9lTrnUbSKWQq4MDNvqexbQPHhJoAtqpW087WSNGy+nZn/r27bf5U/31ZfuEld/15WY49dSKTmfYqiy8UpEfH2XhIAALU3s9fU78jMeyLiUeC1EbFK3QeRF4E/9xPDjAbbak2tb22w77Hy5+Tqxij6th8B7EUxov5KFG9qa5boPympK9zSf5GmNbqv1LoqrFrZ9vby56UDqbQca+GTwJ7A6ylaMbR6n3l7eezStf7UdWrjdWxa2fYPwPzM/GOD8tc2ce4lZObCKMb8uIKi28xOfRTv63/CbRHxLPDGiFg6F2+O/Ww1OdGLdv1PaOdrJWl4DPTe3Yqufy+rsccEhtSkzLwpIn5G0Z3kAxTZ4kZWLn8+0cv+JyimZl2ZxadM/GsfSZGaOQ22/X0A+3oGYSsHZLuGIjt/O8V1PE3R5BiKwUqXGJxJUsfNy8y/DUG9jVp01O4d4yvbVil/PkY/ohgM+AZgc4qWCj+haBr8CkXXiWNo7j5TG8DtneXSmxUr51+WoklzI0/2sn3AMvPKiLgMeE9E7J6Zv+2l6MoUzZ+f6mX/ExT9xSdSPEfNxNiO/wntfq0kDczC8mdfLeNr+xY22DfQe3cruvq9rMYmExhSaz5L8Q3VVyPil72Uqd181wIafXu2dl25muEaIG1PiuTFjzLz4OqOiFibJmdbkTRs+rpH1N7cNvr/vkqDba2ovUkdyLfxH6T4QHxGZh5a3RERG9LkDB4sul9+MTOP769wZr4YES9RjHfRyFpNnr83R1OMcfG1MpnRyJwyjlfROImxNsVrW5+cGq7/Ce1+rSQNTO2+1tcMG2uUP9vddbA/3f5eVmOQY2BILSib836XYtqpw3op9n/lzyn1OyLi9RRN4B4Ygn7sA/X68ufPG+zbfjgDkdQ2s8uf6zbYt8TUqi26ufw5kJHgW7nP1GbhaPTNYe3c2w3g3DX/BywfEW9tsG9KE/X0KjNvB86mmBmlt/7gff1P2IKi9cXt5ZgYneD/BKkz7gZeAjbqY5rQbcqftw1PSD26/b2sxiATGFLrTqTIhH+OsrlynR+UP48rp/0DICLGU4zcPg74/lAH2YcHy59TqhsjYgPg5OEORlJb1MbG+Gg5fSjQ83fdrm/Qf0rRrWH/iPin+p3lQJs1D5Y/p9SV2ZhitqNGat0n1qvfkZkPAr+kmOb5qOo1VureKCKqCZwflj9PKrvO1cqtSdGarl0+D7wAnEDjrha1/wnTIqKnNUwZ0zfK1W78n9DXayVpkMoBes+naDn39YiojjtTu6ceXa6ePbzRdf17WY1BdiGRWpSZz5ZzY3+tl/03RsTXgM8At5fjZrxA8a3lGyn6Gn99uOJt4GLgL8BREbE5RZZ9PeC9FPNuL/HhQVLX+x+KgdHeDdwcEddRNPHdk+Lv+gODPUFmzouID5T1XRQRV5XnXIGiBcJbWdRv+mfA8cDnI2IrivF21gf2oJgadL8Gp6gNFvetiHgbRdPklzOzdq/9N4rWb6cA/xoRN1JMFfpq4A0Ug87twaJB7M6imDllZ+DPEfEbYHmK5+Im2jQwZWY+ERGnlNcLxTeq1f1XRMR3gX8H7oyIn1MMdLcHsDHFQKCntyOWFrXyWklqj09R3Ds/AmwTEVdSTI/6Gor790rAydWZOYbDCHgvqzHIFhjS4Hyb3geHIzP/A9gfuBc4CDic4u/uOGCXDjYVppzaakeKgdreUMa2BcU3bQd2Ki5JrSunmtuNYvrTDYBDKf6+/502fouemdcDW1IkBzaiePO9PzCBogVCrdxzFN/o/4wisXA4RZLjGGBqL3XPoEhSzKboovdFFiUFKKcg3aY8598oEhFHUnRzeKY85oZK+YXAPwFfLuM7nOLN9+nAwS0/CY19jT4G3czMT5TnfIjig8qhwMsUHw72KKc47YhWXitJ7VHe17amaNX7PMV94j8oxtb5HbB7ZrazxVgzsXXte1mNTdH/ALGSJEmSJEmdZQsMSZIkSZLU9UxgSJIkSZKkrmcCQ5IkSZIkdT0TGJIkSZIkqes5jWoT3vOe9+Rll13W6TAkqVOi/yJL8t4paYzz3ilJzWt477QFRhNmzZrV6RAkacTx3ilJzfPeKUlLMoEhSZIkSZK6ngkMSZIkSZLU9UxgSJIkSZKkrmcCQ5IkSZIkdT0TGJIkSZIkqeuZwJAkSZIkSV3PBIYkSZIkSep6JjAkSZIkSVLXM4EhSZIkSZK6ngkMSZIkSZLU9UxgSJIkSZKkrmcCQ5IkSZIkdT0TGJIkSZIkqeuZwJCkUeT2228nInqW8ePH88gjj3Q6LEnqCgcffPBi98hGyzLLLMOkSZPYaqutOOSQQ7j22mvJzE6HLknCBIYkjSpnn332YusLFy7knHPO6UwwkjQCvfLKK8yaNYtbb72VM888kx122IEddtiBBx54oNOhSVJXufPOO/nmN7/JbrvtxqabbsqkSZNYeumlWX311dloo43YZ599+MY3vsH999/ftnOGGeWB22qrrXLGjBmdDkOSGlqwYAGTJ0/mySefXGz7hhtuyD333NOOU0QrB3nvlNQtDj74YH70ox8BsNxyy7H99tsvUWb+/Pk8+uijS7zhnjx5MjfddBOTJ09u9rTeOyWNKv/3f//HcccdxyWXXDKg8hHBdtttxwknnMCUKVMGepqG986lBnq0JKm7XXrppT3Jize84Q3MnTuXRx55hHvvvZff//73vPOd7+xwhJLUPdZcc00uu+yyXvfff//9HH300fziF78A4NFHH+XII4/kZz/72XCFKEld55RTTuEzn/kMCxcuXGz7aqutxuTJk1l99dWZM2cOTz75JI8//jgAmcl1113HDjvswMknn8xnPvOZls9vFxJJGiWq3UcOOuggDjjggIb7JEn922CDDfjpT3/Kjjvu2LPtV7/6FbNmzepgVJLUOYcddhif/vSne5IXyyyzDEcccQQzZsxg1qxZ3HbbbVxzzTXceuutPPbYY9x///2cdtppvO51r+upo5bUaJUJDEkaBZ599lkuvvhiAMaNG8eHPvQhPvzhD/fsv/DCC5k/f36nwpOkEWncuHEcfvjhPesLFizg1ltv7WBEktQZZ511FqeffnrP+kYbbcTtt9/Oqaeeylve8hYiluzx8drXvpbDDz+cu+66i9NPP50JEyYMOg4TGJI0Cpx33nm8/PLLAEyZMoXJkyez2Wab8Q//8A8AzJ07l1/+8pedDFGSRqRNNtlksfVnnnmmQ5FIUmfcd999iyVzN9hgA2644QY23HDDAR2/9NJL84lPfIKbbrppsdYYrTCBIUmjQLWLSLXlRfWx3UgkqXm15HDNiiuu2KFIJKkzvv71r/e05B03bhxnn302kyZNarqeLbbYgsMOO2xQsZjAkKQR7o477qA2Uv3yyy/P+9///p59+++/P+PHjwfg6quv5tFHH+1IjJI0Ut18882Lrb/xjW/sUCSSNPxmzZrVM3sTwK677sp2223XsXhMYEgjzPTpxSLVVFtW7Lnnnqy00ko962uttRY777wzAAsXLuScc84Z7vAkacR65plnOOmkk3rW3/72t7PBBht0MCJJGl5XXHEFL774Ys/6v/3bv3UwGhMYkjSiLViwgB//+Mc969UuI422VTPokqQlvfTSS9x3331873vf4y1veQv3338/ACussAJnnHFGh6OTpOF1/fXX9zyOCHbYYYcORgNLdfTsklpWbYUxdWrn4lBnXXbZZTzxxBMATJo0iXe9611LlNlrr71YccUVef7557nnnnu48cYbecc73jHcoUpSV3nooYcajprfyJQpU/jWty9h77sAACAASURBVL7FlltuOcRRSVJ3qXVThmLmkYkTJ3YwGltgSF3N7iLqT7X7yP77789SSy2Zl54wYQJ77bVXz7qtMCRp4Lbddls+9rGPOfaFpDHp6aef7nm83nrrdTCSgi0wpC5jwkIDNXv2bC6++OKe9QMPPLDXsgceeCDnnnsuABdccAGnnXYayy233JDHKEndarnllmP77bdfYvvChQuZM2cOd999N3PmzOGGG27ghhtu4Mtf/jIXXnghm266aQeilaTOqE4dvfLKK3cwkkLHW2BExOsj4nsRcVtELIiIaxuUeTAism55skG5zSLi6oiYFxGPR8SJETG+rkxExLER8UhEzI+I6yLC9oCSRpzzzjuPl156CYCNN96Yt771rb2W3WmnnVh77bUBmDNnDr/85S+HJUZJ6lZrrrkml1122RLLFVdcwR/+8Admz57N5ZdfzmabbQbA7bffzpQpU3jooYc6HLkkDZ/ae02AZZddtoORFDqewADeAOwG3FMuvfkJsE1l2a26MyJWBa4CEtgTOBH4FHBCXT2fBT4PnAzsATwPXBURaw32QqTBsLuImlXtPtJo8M6q8ePH86EPfajhsZKkJUUE73rXu7j++ut7mk3/9a9/5ZBDDulwZJI0fFZZZZWex3Pnzu1gJIVuSGBcnJnrZua+wB19lHsiM2+uLP9bt//jwPLA3pl5ZWaeSZG8OCoiJgJExHIUCYyvZubpmXkVsC9F0uPQdl+YNFxMfow9d911F3/84x971r/61a+yyiqr9Ln853/+Z0/5q666iscee6wToUvSiLLaaqtx5JFH9qxfeuml3HfffR2MSJKGz2qrrdbz+Nlnn+1gJIWOJzAyc2GbqtoVuDwzq2mh8ymSGrUOju8AJgIXVs7/AnBxebwkjQg//OEPF1t/4YUXmDNnTp/LvHnzesovXLiQc845Z7jDlqQRadttt11svTqtoCSNZhtssEHP4zvuuIPM7GA0XZDAaMK/RMTLETEnIn4WEa+p278JMLO6ITMfBuaV+2plFgD31h17V6WMJHW1BQsW8N///d+DrsfZSCRpYKpNqAGefHKJodgkaVTabrvteh4/99xz3H333R2MZuQkMC4CPgHsBBxNMQbG9RFRHQZ1VeC5BsfOLvfVyjyfmQsalJkQEcvUHxwRUyNiRkTMqE4hI0mdcsUVV/DEE08AxdgWTz/9NJk5oOXRRx8lIgC4++67ufnmm4ckRu+dkkaT2bNnL7a+/PLLD8l5vHdK6jZTpkxZbP3888/vTCClEZHAyMwjMvO8zLw+M6cD7wZeDXykvmiDw6Nue29lGu7LzOmZuVVmbjVp0qQWopek9qoOwDllyhTWWGONAR+7zjrr8I53vKNhXe3kvVPSaHLdddcttv6a19Q3BG4P752Sus0222zD5ptv3rN+1llnMX/+/I7FMyISGPUy83bgbuDNlc2zgVUaFF+ZRS0zZgMr1U+tWh43LzNfaXesktROzz33HBdddFHP+j777NN0Hfvuu2/P4wsuuIAXX3yxLbFJ0mg0a9YsTj311J71ZZZZhh133LGDEUnS8Pr0pz/d8/ixxx7j2GOPbbmue++tH82hOSMygVFRbTExk7pxLCJiXWAFFo2NMRMYD7y+rp4lxs+QukltlhFnGtF5553XMx/3+PHj2XvvvZuuY5999unpRvLcc8/xq1/9qq0xStJokJlcccUVbLfddovN2nTYYYcxceLEDkYmScPrgAMOYPvtt+9ZP+200zjjjDOaqmP+/PkcdNBBTR9Xb6lBHd0hEfFGYGPge5XNlwJHR8RKmfm3ctt+wHzgd+X6jcBciqlTv1TWNQHYA/CjoUa8aoJj6tTOxaGhU+3ysd122/GqV72q6TrWWWcdttlmG2688caeOj/4wQ+2K0RJGhGeeuop3vOe9yyxfeHChcydO5eZM2cyZ86cxfb94z/+IyeeeOJwhShJXWH8+PGcd955bLnllvz1r38lMzn00EP505/+xBe/+EXWWmutXo9dsGAB55xzDtOmTePhhx/miCOOGFQsHU9glAmE3crVdYCJEVFrE30JsANwIPAb4HGK1hLHAQ8DZ1eqOhM4HPhFRJwMbABMA75Zm1o1M1+MiJOAz0fEbIpWF0dRtET5zhBdoiS1xcyZM7nlllt61qtdQZq177779iQwrrzySh577DHWWWedQccoSSPFiy++yOWXXz6gsuPGjePQQw/lK1/5ChMmTBjiyCSp+6y99tpce+217L777jzwwANAMR7Gj3/8Y3bZZRd23nln1ltvPVZbbTXmzJnDE088we9//3suueQS2jkocccTGMCrgJ/WbautvxZ4pCxzKsVYFc8AlwHH1hITAJk5OyJ2Ak4HLqYY9+JbFEmMqpMoEhbHAKsDM4BdMvOp9l2SNDB2CVEzqq0vxo0b11L3kZp99tmHo446isxk4cKFnHvuuXz2s59tQ5SSNLJFBCussAKrrbYam2++Odtuuy0HHnggkydP7nRoktRRm266KTfffDNTp07tGZNt/vz5/PrXv+bXv/51v8fvsssuTB1kM/HIbDQphxrZaqutcsaMGZ0OQ6PIUCYw7EKiIRD9F1mS905JY5z3Tkmjzh/+8AdOOeUUrrjiiiW621WtvPLK7L333nz0ox/lne98ZzOnaHjv7IYWGJIkSZIkaYTYeuutufDCC1mwYAEzZszggQce4Omnn2bu3LmstNJKrLHGGrzpTW9i0003Zdy49s0dYgJDkiRJkiQ1bfz48Wy99dZsvfXWw3K+kT6NqiRJkiRJGgNsgSF1gIN3SpIkSVJzbIEhSZIkSZK6ngkMaZSaPt2WHpIkSZJGD7uQSMPEZIIkSZIktc4WGJIkSZIkqevZAkMa5aotP6ZO7VwckiRJkjQYTbfAiAiTHpIkSZIkaVi10oXkkYj4ckS8tu3RSJIkSZIkNdBKAmNZ4Bjg3oi4NCL2jAjH0pAkSZIkSUOmlcTD2sDBwM3Au4FfULTKOCEi1m1jbJLazKlVJUmSJI1UTScwMvOlzDwnM7cF3gCcDiwHfB64PyJ+HRG7R0S0OVZpRDJpIEmSJEmDN6iuH5l5V2YeAbyaRa0y3gv8GngwIo6LiDUHHaUkSZIkSRrT2jJ2RWa+BPwSOA94DAhgXeBEikTGNyJimXacS5IkSZIkjT2DnhI1IrYCPgZ8EJgAPA98F/gB8Gbgk+WyHHDoYM8nafCqXVqmTu1cHJIkSZI0UC0lMCJiBeAAisTFlhQtLv4M/Cfw35n5Qln0fyPih8AVwH6YwNAY4rgXkiRJktQ+TScwIuJMYH9gReAVim4j383MGxuVz8wFEXENMGUQcUqSJEmSpDGslRYYU4EHga8A38/MWQM45ndleUmSJEmSpKa1ksDYA7gkM3OgB2TmDcANLZxL0hCrdXVxLAxJkiRJ3azpBEZm/nYoApEkSZIkSepN09OoRsQOETE9ItbuZf+ry/3/OPjwJEmSJEmSWutCcjjwhsx8otHOzHy8TF6sClw3mOAkSZIkSZKghRYYwFvofzyLG4C3tVC3JEmSJEnSElppgfEq4PF+yjxZlpPGlNqAmJIkSZKk9mqlBcYcYHI/ZSYDL7RQtyRJkiRJ0hJaSWD8EXhfRKzZaGdErAW8rywnSZIkSZI0aK0kME4HJgLXRcRuEbEUQEQsFRG7A78DVgK+074wJUmSJEnSWNb0GBiZeVlEfBU4BrgYWBgRs4A1KBIiAXw1My9pa6SSJEmSJGnMamUQTzLzcxHxe+AwYGuKATvnADcD38nMS9sXoqThUB2AdOrUzsUhSZIkSY20lMAAKFtY2MpCkiRJkiQNuVbGwJAkSZIkSRpWLbfAAIiIZYFVgPGN9mfm44OpX5IkSZIkCVpMYETE/sB/AG+kGLSzkWy1fkmSJEmSpKqmEwwR8WHgR8BCikE7HwH+3ua4pBGlOgCmJEmSJKn9Wmkh8RmKGUe2y8zb2xyPJEmSJEnSEloZxHND4EKTF5IkSZIkabi0ksCYDcxvdyCSJEmSJEm9aSWB8VtgSkT0NninJEmSJElSW7WSwPgssAJwRkRMaHM8kiRJkiRJS2hlEM+fUAzi+THggIi4G3iuQbnMzHcPJjipmznziCRJkiQNn1ZaYOwMvBkIYCVgq3Jbo6VfEfH6iPheRNwWEQsi4toGZSIijo2IRyJifkRcFxFbNii3WURcHRHzIuLxiDgxIsa3Upc0lk2fboJGkiRJUndpJYGx9ACXZQZY3xuA3YB7yqWRzwKfB04G9gCeB66KiLVqBSJiVeAqIIE9gROBTwEnNFuXJEmSJEnqLk0nMDJzwUCXAVZ5cWaum5n7AnfU74yI5SiSDl/NzNMz8ypgX4pExaGVoh8Hlgf2zswrM/NMiuTFURExscm6JEmSJElSF2mlBUZbZebCfoq8A5gIXFg55gXgYmDXSrldgcszc25l2/kUSY3tm6xLkiRJkiR1kZYSGOU4EodExA0R8UxEvFjZt2VEfDsiNmxTjJsAC4B767bfVe6rlptZLZCZDwPzKuUGWpckSZIkSeoiTScwImJp4HLgdGAz4CWKMS9qHgKmAge0I0BgVeD5Bl1SZgMTImKZSrlGs6HMLvc1U1ePiJgaETMiYsbTTz/d8kVI0ljivVOSmue9U5L61koLjE9TzDDyJWASsNhcBZk5G7geaOcUqtlgWzTY11u5gZRpuC8zp2fmVpm51aRJkwYSqySNed47Jal53jslqW+tJDAOBG7KzC+ULRkaJQTuB14zqMgWmQ2sVD8dKrAKMC8zX6mUW6XB8SuzqGXGQOuSJEmSJEldZKkWjtmAovtIX54FVm+h7kZmAuOB1wN3V7bXj3kxk7pxLCJiXWCFSrmB1iUtZnqlndHUqZ2LQ5IkSZLGqlZaYLxI0aqhL+vReDyKVtwIzKWY7hSAiJgA7AFcWil3KfDuiFipsm0/YD7wuybrkno1ffriCQ1JkiRJ0tBrpQXGn4BdImKZzHy5fmdETATeBdw8kMrKBMJu5eo6wMSI2KdcvyQz50XEScDnI2I2RUuJoyiSL9+pVHUmcDjwi4g4maKlyDTgm7WpVTPzxQHWJQlbnkiSJEnqHq0kMM4CzgV+FBH/Vt1RJi9+AKwGfG+A9b0K+Gndttr6a4EHgZMokgzHUHRNmQHskplP1Q7IzNkRsRNF95aLKVqAfIsiiVHVb12SJEmSJKm7NJ3AyMwfR8S7gA8D76MYGJOIuBnYHFge+F5m/maA9T3IollAeiuTwJfLpa9ydwI7tqMuSZIkSZLUPVoZA4PM/GdgKvAXYC2KBMTbgIeBj2XmIW2LUJIkSZIkjXmtdCEBIDPPAs6KiBUpuozMycw5bYtMUlepjYfhWBiSJEmSOqHlBEZNZj4PPN+GWCRJkiRJkhpqqQuJJEmSJEnScGq6BUZE3DPAopmZGzdbvyRJ0mBNn26XN0mSRptWupBMALLB9pWBFcvHTwF/bzUoSZIkSZKkqlamUZ3c276I2AQ4DVga2HUQcUmSJEmSJPVo6xgYmTkT2AtYH/h8O+uWJElql9rMSpIkaeQY9Cwk9TJzXkRcDhwIHNfu+qXh5BtcSZIkSeoOQzULySvAWkNUt6QOmj7dxI6k7tPbfcl7liRJo0fbExgRsRpFN5JH2123JElSK0xiSJI08rUyjeqxfdS1LkXyYlXsPiJJkrqMiQxJkkauVsbA+FI/+58HTsrMr7ZQtyRJ0pCoT140SmZMnTo8sUiSpOa1ksDYpZftC4HZwJ2Z+XLrIUmSJA2crSokSRobmk5gZObVQxGIJEnSQEyfXrSUMHEhSdLY0vZpVKWRzjfEkjQy1e7ftW4grd7P6+uRJEndoZVBPF/d6sky8/FWj5UkSRoIE9GSJI1OrbTAeBTIFo7LFs8nqQtVPyD4LaWk4WJyQpKksauVhMJPgPWAbYG/AX8GngTWArYAVgKuBx5uU4ySJGmMM3EhSZJaSWCcANwEfAf4QmY+V9sREasAXwT2Bz6amX9pS5SSJGnMMnkhSZIAxrVwzMnAXZl5RDV5AZCZz2XmYcDMspwkSdKIYbJEkqTu1UoCY3vgun7KXFeWkzQGTJ/um35Jw2s47jne1yRJ6i6tJDCWBdbsp8xawHIt1C1JkiRJkrSEVsbAuA34YER8OzP/XL8zIrYE9gP+d7DBScPJb9okSZIkqXu1ksA4EfgtcEtEnEPRXeQpilYZ2wMfLus9sV1BSpKksaObp2muxdZtcUmSNBY03YUkMy8HDgDmA/8K/Ai4rPz50XL7AZl5ZRvjlCRJGna2zpMkqXu00gKDzLwgIi4B9gLeDKwMzKHoNvLLzPxb+0KUJEljVbckELolDkmSxrJWBvEEIDP/lpnnZOaRmfmR8uc5Ji8kSdJAVRMDJgkkSVJfWmqBURURKwErZuYTbYhHGla+WZYk9cf/FZIkdYeWWmBExAoRcXJEPAo8BzxS2fe2iPh1ORuJJElSn6ZPN0kgSZL613QLjLLFxQ3A5sDtwFxg40qRO4AdgZnAn9oQoyRJkiRJGuNaaYFxHEXy4l8zcwvgwurOzHwB+B2w0+DDkyRJY4ktMSRJUm9aSWC8H7giM39QrmeDMg8Ck1sNSpIkSZIkqaqVBMZk4LZ+yjxPMbWqJElSQyO5tcVIjl2SpJGqlVlIngcm9VPmtcCsFuqWNIJV39BPndq5OCRJkiSNPq20wPgj8N6IWLHRzohYC9gVuHEwgUlDyRHvJUnt4v8TSZKGRysJjG8DawC/iYgNqzvK9QuA5ctykiRJo55JDEmShl7TXUgy89KI+BLFbCQzgZcAIuJJiq4lAXwuM29oZ6CSJEndxKSFJEnDq5UxMMjM4yPieuBw4O3AsuVyBfDNzLyyfSFKkqTRxA/+kiSpFS0lMADKJIWJCkkN1T6gOJinJEmSpHZoegyMiLgiIqYNQSySJGkUG+0tLxwgWpKkodXKIJ7bAsu0OxBJkjT6+QFf0kg32u5jo+16NLq1ksD4C7BuuwORJEkaDWyJIUnS0GglgfF9YLeImNzuYHoTEQdHRDZYPl4pExFxbEQ8EhHzI+K6iNiyQV2bRcTVETEvIh6PiBMjYvxwXYs01tTeyPtmXpIkjWbV9zp9ve8ZC++LRvv1qXNaSWD8HPg98PuI+HhEvCUi1omIV9cvbY4VYEdgm8ryi8q+zwKfB04G9gCeB66KiLVqBSJiVeAqIIE9gROBTwEnDEGskiSp5JtZSVpcO+6L7U6GjMR79VhICGmRVhIYDwO7U3QjOQO4pdz2SN3ycJtirPpjZt5cWf4KEBHLUSQwvpqZp2fmVcC+FImKQyvHfxxYHtg7M6/MzDMpkhdHRcTEIYhXkiRJ0igzlB+YB9qSo9v1F/tgWqn0ts9kxujXSgLjJ+Xy48rjRst5bYpxIN4BTAQurG3IzBeAi4FdK+V2BS7PzLmVbedTJDW2H4Y4JUnSGOGbaGnsGYoP3o22N3N/6a3udt2javUMtj7vmRqIpZo9IDMPHIpABui+iFgduA/4ZmZ+r9y+CbAAuLeu/F3AfpX1TYBrqgUy8+GImFfuu3hIolZH1W6GU6d2Ng5J0tg1fbr/h6SRpvoecrg+XDebmBiJmo27t/tns8+V9+DRoZUWGJ3wBMX4Fh+mGN/iD8CZEfHJcv+qwPOZuaDuuNnAhIhYplLuuQb1zy73LSEipkbEjIiY8fTTTw/yMtRJDiYpDR/vndKS/P+j/njvHLmGsoVDfV29navZMgPdNxjtqreddVRjqn4+8HPCyDCgBEZEHBQRWwx1ML3JzMsz80uZeUVmXpqZB1F0FzkuImrXkA0OjQb7eivXaDuZOT0zt8rMrSZNmtTqJUjSmOK9U+AbQqlZ3jvVl1bup63cfweSCBlMnc3U1cz5m03aNHNOdY+BtsA4G3hfdUNE/HNEXNO4+LD4GbAasD5FC4qVGkyHugowLzNfKddnl9vqrUzjlhmSJKkFvgGUNFoM14Cd3aqZVhvNjuUx2PM1W89IeL7Vt6bHwKhYn+4Y+DKBmcB44PXA3ZV9m5T7amaW23pExLrACnXlJEmSJI1R7R7gcrgMZAaTVmc5aWVw0mbLmWBQfwaTwOi09wOzgIcoxsiYSzF16pcAImICxXgZ1T+DS4GjI2KlzPxbuW0/YD7wu2GKW5IkSdIIN5QDQ3b6g3yjlhV9rbfzXFJfRkQCIyJ+DtwC/JmipcV+5XJ4Zi4EXoyIk4DPR8RsitYUR1F0kflOpaozgcOBX0TEycAGwDSKGU2qU6tqhPNGKEmd4f1XkiQNlRGRwKDoFvIvwLoUA27eCRyUmedWypxEkbA4BlgdmAHskplP1Qpk5uyI2Ak4nWLK1OeAb1EkMSQNMae0lUY3kxeS1DzvnZ3h8z4yNZPAaDhLx3DIzGOBY/spk8CXy6WvcncCO7YvOkmSpMZ8gyyNbv6Ndydfl9GrmQTGtIiYVr8xIhb0Uj4zc6S08JAkSZIkSV2smQRDNFl3s+UlSZIkSZIaGlACIzPHDXUgksaOarM+x8OQRgeb6w6MYwFJktQ6ExOSJKkl06ebuJAkScPHBIYkSRoUkxjN8zmTJKl5JjA0qvhtoCRppPD/lSRJzTGBIamjTDpJI5N/t5IkabiZwJAkSeoQE0GSJA2cCQxJkiRJktT1TGBIkiRJkqSuZwJDkiQNmF0eho7PrSRJfTOBIUmSmuIH7aHjcytJUu+W6nQAkgSLv2mfOrVzcUhSp9Xuh94LJUlanC0wJEmSJElS17MFhkY8m9tKkkYy/49JkjQwJjAkSVK//JAtSZI6zS4kkiRJkiSp65nAkNR1pk/3216pm/j3KEmSuoEJDEmStAQTiZ3n8y9J0uIcA0Mjlm/sJGloeH+VJEndyBYYkiSpVyYzOs/XQJKkgi0wJEmSulQteVFNYkyd2plYJEnqNFtgSJIkSZKkrmcLDI0oNqMdW/zGUZIkSVKNCQxJkgSYJJYkSd3NLiQaEZzOT/4OSFLBe6EkaayyBYYkSWOcH4hHntprZvc6SdJYYgsMSZLGMJMXI5+voSRprLAFhrqab8pUz4E9pfbxHitJkkYSW2BIkiSNUCahJEljiS0w1HV8MyZJUnOmT7dVmiRp9DOBIWnEchA7qTUmiiVJ0khkFxJJksaAWtLC5MXo52ssSRqtbIGhruEbLkkaWt5nJUnSSGYLDEmSpFFg+nRb2kiSRjdbYEiSNIr5QXbscpwgSdJoYwsMSZIkSZLU9WyBoY7ym0G1Q3+/R377KEmSJI18JjA0bKofMv1AqeFkM2pJkiRp5DOBoY6w5YUkSZIkqRmOgSFJkiRJkrqeLTA05GxtoW5hVxJJkiRp5DKBoSFh0kLdzPFYJEmSpJFnTCYwImIz4DvANsBzwFnACZm5oKOBjQImLiRJkiRJQ2HMJTAiYlXgKuBOYE/gdcApFOOBHNfB0EYckxUaDfr6PbZ1hiRJktQ9xlwCA/g4sDywd2bOBa6MiInAtIj4WrlNkiRJkiR1kbGYwNgVuLwuUXE+cDKwPXBxR6Lqcra20Fg00N97W2pIkiRJQ28sJjA2Aa6pbsjMhyNiXrlvVCQw+pptwWSE1F6N/qaqf3t2U5EkSZIGLzKz0zEMq4h4BTg6M0+t2/4ocE5mHlu3fSpQ+4ixMXD3sATa2BrArA6ev9O8/rF9/eBz0Onrn5WZ7xlIQe+dXcXrH9vXDz4Hnb5+750jk9fv9Y/l64fOPwcN751jNYHx6cw8rW77Y8DZmfm5zkTWv4iYkZlbdTqOTvH6x/b1g8/BWL/+Vo31583rH9vXDz4HY/36WzXWnzev3+sfy9cP3fscjOt0AB0wG1ilwfaVKaZUlSRJkiRJXWYsJjBmUox10SMi1gVWKPdJkiRJkqQuMxYTGJcC746IlSrb9gPmA7/rTEgDNtaH3/T6Ndafg7F+/a0a68+b16+x/hyM9etv1Vh/3rz+sW2sXz906XMwFsfAWBW4E7idYurUDYBvAv+/vXuPtaws7zj+/QlYRbmMKGAollFRQG0EoQYMgiVBsOk4FcFopkYUgjQVVCyKoWWkqXfbUbFpuAj9o9QUDQJGEalKMzUySmgCiUQRrygIMuEiCDg8/WOtM7PPZs/h7OGcvfbl+0km65x3vXvvZ5+T/XtPnlmXdVV1dpe1SZIkSZKkwWaugQGQ5ADgPOBQmuteXAisrapNnRYmSZIkSZIGmskGhiRJkiRJmiyzeA2MqZPk3UkqyRe7rmUUkuyc5ENJNiS5N8kdSS5P8qKua1sOSQ5I8t9JHkzyqyTnJtmu67pGIcnxSa5McnuSB5LckOTNXdfVlSR7tT+HSvLMruuZdGan2TmtzM75zM6lZXaandPK7NxinHPTBsaES7I78A/AXV3XMkLPA04Gvg68ETgFeC5wfXtHmanRXrPlWqCA1wPnAmcAH+qyrhF6L/AA8B5gFfAt4NIk7+q0qu58gubnoSfJ7DQ7p5zZOZ/ZuUTMTrNzypmdW4xtbnoKyYRLchHwVGBv4O6qemPHJS27JM8AHquqh3rGngX8HPhEVU1NyCY5CzgT+JOquq8dOxNYC+w5Nzatkjy7qu7uG7sUOLSqVnZUVieSHA5cAXyYZlHZqarGcmGZBGbn5jGzcwqZnVuYnUvL7Nw8ZnZOIbOzMe656REYEyzJIcAJwAe6rmWUqup3vYtIO3YP8DNg926qWjbHAl/vWzC+ADwdOKKbkkanfxFp3cj0/Z4X1B66+Vma/wkZ9DPREMzOeWNm5xQyOxtm59IyO+eNmZ1TyOycjNy0gTGhkoTmTiofr6rbu66na0meA7yQ5ha502Q/4Jbegar6OfBgu28WHcb0/Z6fyDuBpwGf67qQSWd2zmd2zhSzU9vM7JzP7Jwps5adY5+b23ddgLbZicCewCe7LmRMfIrmPK0vdF3IEltBc6vffhvbfTMlyVE052S+vetaRiXJbsA/AmuqGsCMAAAAB8RJREFU6tHmb0g9CWbnfGbnDDA7zc4lYHbOZ3bOgFnLzknJTRsYYyLJLjQXBFpQVd3Szv0wcFr/IW2Tapj3P+CxpwJrgOOq6rfLUF7XBl2oJlsZn1pJ9gEuBa6oqks6LWa0/gm4vqq+2nUh48jsNDsXYHZidpqdg5mdZucCzE5mNjsnIjdtYIyP44ELFjEvwAeBXwDXJNm1Hd8e2KH9/v6q2rQ8ZS6bYd7/lm+SVTTnab2/qi5fjsI6thHYdcD4LgzukE+l9mJZX6O5YNaajssZmSQvoen6v7rns75ju90lyaZp+WPySTA7zc5BzE7MTszOhZidZucgZiezmZ2TlJvehWQCJfkyzeFMW3N4Va0fVT1dSXIYza2ePl9Vf9t1Pcshyf8At1fVm3vG9qYJ1FVVdVVnxY1Ikh1pfs970FwF+jcdlzQySVYDC/2BdFFVnTSqeiad2dkwO83OaWd2Li2zs2F2mp3TbJJy0yMwJtPZwLq+sXXAvcA5wE0jr2jE2i7hV4CrgdM6Lmc5fQ34uyQ7VdX97dibgIeA67orazSSbA9cBuwLvGpWFpEe64HX9I0dA7wfeB1w28grmmxmp9lpds4Gs3NpmZ1mp9k5/SYmNz0CY0ok+Tazcz/u3YEbaM7Feyvw+57d91XV1FwpOMkKmisf3wx8DHg+8M/Auqo6u8vaRiHJ+cDJwOnAhr7dN1bVw6OvqltJ3gZczJjdk3tSmZ2bmZ1TxOx8PLNzaZmdm5mdU8TsnG9cc9MjMDSJDgD+uP36W337rgOOHGk1y6iqNrZXQD4PuIrm/MN/AdZ2WdcIHd1uPz1g30rgp6MrRZp4ZqfZCWanNCyz0+wEs3NseASGJEmSJEkae0/pugBJkiRJkqQnYgNDkiRJkiSNPRsYkiRJkiRp7NnAkCRJkiRJY88GhiRJkiRJGns2MCRJkiRJ0tizgSGNkSQvTFJJLuy6FkmaFGanJA3P7NQksoEhLUKSS9uAP3URc7/Rzl09itokaVyZnZI0PLNT2jobGNLinN9uT15oUpJ9gKOAXwNfWd6SJGnsmZ2SNDyzU9oKGxjSIlTVt4EfAgcmOWiBqe8AAlxcVX8YRW2SNK7MTkkantkpbZ0NDGnxLmi3A7vhSbYDTgQKuLAd2yvJOUm+k+SOJI8kuT3JfyTZb7EvnGR9koELU5KT2kMH1wzYt3eSf01yW5KHk/w2yRVJXjFg7s5trTcnub/9d2uS/0xy4GJrlaQ+ZqckDc/slAawgSEt3r8DjwBvSbLjgP3HAnsB11bVT9qx1wBnAvcAXwLWARuAE4ANSV66XMUmORj4P+CdwC3AZ4CrgCOB7yQ5umdugGuAtcC9NIvmvwHfa9/DK5erTklTz+yUpOGZndIA23ddgDQpququJF+mWQROAC7pmzLXIT+/Z+wbwB5V9UDvxLazvB74CPCXS11rkh2A/wJ2BF5dVet79p1Ns0B8Psnzq+oR4OU0i8UXq+r4vufaDth5qWuUNBvMTkkantkpDeYRGNJw5haJk3oHkzwXeB1wJ3DF3HhV3dm/iLTjNwLXAUe1Qb3UVgErgXW9i0j72r8EPknTtT+y73EPDah1U1VtXIYaJc0Os1OShmd2Sn08AkMazjeBHwOvSrJ/Vf2gHT+R5vN0SVU92vuAJKuAU4BXALvx+M/ds4C7lrjOQ9vtyiRrB+x/cbvdn+YQvpvaf3+dZCVwJU2n/vv970eStoHZKUnDMzulPjYwpCFUVSW5kOYQvJOAM9rz+N5Oz0WU5iR5L/ApmnMRrwV+RtNtLuANwMuAP1qGUndrt296gnnPBKiqPyQ5EjgHOA74eLv/viSXAB+sqt8tQ52SZoDZKUnDMzulx7OBIQ3vYuBc4K1JzgIOB14AfLOqbp2b1J4PuBb4FXBQVd3Z+yRJDh/iNR9rHpKnVNVjfft2HTD/3nb7F1X11cW8QFXdA5wOnJ5kX5rD/E4BTqM5F/HEIeqVpH5mpyQNz+yUengNDGlI7YJwJfBsYDVbzks8v2/qHsBOwPoBi8jOwDC3iNpI83nda8C+gweMfbfdDrNYbVZVP6qqC4AjaDr3q7fleSRpjtkpScMzO6X5bGBI22bu3txnAH8F3A1c3jfn18DvgUOSPGNuMMlTgc8CK4Z4vQ3tdt69wNtbUh3/+OlcDvwUOC3Jawc9YZLDkjyt/foFSfYfMG0FsAPw4BC1StLWmJ2SNDyzU2p5Com0ba4BfgL8Wfv9ee1toTarqk1JzgPeB9yU5Eqa8w7/HNiF5mrQRyzy9S6iWbT+vr0V1g+A/YBjaBaN4/pe++EkbwCuBq5O8r809+Z+CHgecAjN1aKfQ7PYHQhcluT7wM00i+DuwOtpcuJji6xTkhZidkrS8MxOqeURGNI2qKqiCfc5F2xl6lnAmcDDNOf1rQaupwnyXw7xenfQLDpX05wj+Dc0hwke1Y4NesyNwJ/SXBhpBc0Fn04FDgJuANbQHCJIW9NHgUeBY2kWrdfSdOCPqarPLLZWSdoas1OShmd2Sluk+TxIkiRJkiSNL4/AkCRJkiRJY88GhiRJkiRJGns2MCRJkiRJ0tizgSFJkiRJksaeDQxJkiRJkjT2bGBIkiRJkqSxZwNDkiRJkiSNPRsYkiRJkiRp7NnAkCRJkiRJY+//ATxIDVhvoQHwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt \n",
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "# === Noramal and Truncated normal distributions ===\n",
    "mean = 0\n",
    "std = 1\n",
    "x_normal = tf.random.normal((1,50000),mean,std)\n",
    "x_truncated = tf.random.truncated_normal((1,50000),mean,std)\n",
    "\n",
    "# === Uniform distribution\n",
    "minval = -2 \n",
    "maxval = 2\n",
    "x_uniform = tf.random.uniform((1,50000),minval,maxval)\n",
    "\n",
    "\n",
    "def simpleaxis(ax):\n",
    "    ax.spines['top'].set_visible(False)\n",
    "    ax.spines['right'].set_visible(False)\n",
    "    ax.get_xaxis().tick_bottom()\n",
    "    ax.get_yaxis().tick_left()\n",
    "#     ax.set_ylim([-1.1,1.1])\n",
    "    ax.tick_params(axis='both', which='major', labelsize=15)\n",
    "    \n",
    "def get_axis_limits(ax, scale=.8):\n",
    "    return ax.get_xlim()[1]*scale, ax.get_ylim()[1]*scale\n",
    "\n",
    "f,axarr = plt.subplots(1,3,figsize=[15,4],sharey=True)\n",
    "titles = ['Normal','Truncated Normal','Uniform']\n",
    "\n",
    "print(x_normal.shape)\n",
    "for i,x in enumerate([x_normal,x_truncated,x_uniform]):\n",
    "    ax = axarr[i]\n",
    "    ax.hist(x[0],bins=100,color='b',alpha=0.4)\n",
    "    ax.set_title(titles[i],fontsize=20)\n",
    "    ax.set_xlabel('Values',fontsize=20)\n",
    "    ax.set_xlim([-5,5])\n",
    "    ax.set_ylim([0,1800])\n",
    "    \n",
    "    simpleaxis(ax)\n",
    "    \n",
    "    \n",
    "axarr[0].set_ylabel('Frequency',fontsize=20)\n",
    "plt.suptitle('Initialized values',fontsize=30, y=1.15)\n",
    "\n",
    "\n",
    "for ax,letter in zip(axarr,['A','B','C']):\n",
    "    simpleaxis(ax)\n",
    "    ax.annotate(letter, xy=get_axis_limits(ax),fontsize=35)\n",
    "\n",
    "plt.tight_layout()\n",
    "\n",
    "# plt.savefig('histograms.png', bbox_inches='tight', format='png', dpi=200, pad_inches=0,transparent=True)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `Batch Normalization`\n",
    "- 设定合适的初始值，则各层的激活值分布会有适当的广度，从而进行顺利学习\n",
    "- “强制性”的调整激活值的分布，`Batch Normalization`\n",
    "    - 在神经网络中插入对数据分布进行正规化的层，即 `Batch Normalization` 层，以进行学习时的 mini-batch 为单位，进行正规化\n",
    "    \n",
    "    \n",
    "1. $\\mu_B=\\frac{1}{m}\\sum_{i=1}^{m}x_i$\n",
    "2. $\\sigma_B^2=\\frac{1}{m}\\sum_{i=1}^{m}(x_i-\\mu_B)^2$\n",
    "3. $\\hat{x_i}=\\frac{x_i-\\mu_B}{\\sqrt{\\sigma_B^2+\\varepsilon}}$\n",
    "    - $\\hat{x_i}$ 以 0 为均值，方差为 1；$\\varepsilon$ 微小值（如，$10^{-7}$），避免被 0 除，平滑项. \n",
    "4. $y_i=\\gamma \\hat{x_i}+\\beta$\n",
    "    - 对正规化后的数据进行缩放和平移变化，初始时$\\gamma=1, \\beta=0$ ，再通过学习调整到合适的值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Batch Norm 有以下优点：\n",
    "- 可以使学习快速进行（可以增大学习率）\n",
    "- 不那么依赖初始值，\n",
    "- 抑制过拟合（降低 `Dropout` 等的必要性）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 正则化\n",
    "### 过拟合\n",
    "- 只能拟合训练数据，不能很好的拟合不包含在训练数据中的数据\n",
    "- 原因：模型拥有大量参数、表现力前；训练数据少\n",
    "\n",
    "\n",
    "### 权值衰减\n",
    "- 在学习的过程中对大的权值进行惩罚，来抑制过拟合；很多过拟合原本就是因为权重参数过大导致的\n",
    "- 例如为损失函数加上权重的平方范式(L2范数)$\\frac{1}{2}\\lambda W^2$，误差反向传播时的梯度也要加上$\\lambda W$\n",
    "- 除了 $l2$ 范数，还有 $l1$ 和 $l\\infty$范数都可以作为正则项\n",
    "\n",
    "### `Dropout`\n",
    "- 在学习的过程中随机删除神经元的方法，被删除的神经元不再进行信号的传递；测试时，会传递所有的神经元信号，但输出会乘上训练时的删除比例后再输出\n",
    "- 训练时每次更新，都会随机删除神经元，相当于生成一个新的神经网络，`Dropout` 将集成学习的效果通过一个网络实现了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dropout:\n",
    "    def __init__(self, dropout_ratio=0.2):\n",
    "        self.dropout_ratio = dropout_ratio\n",
    "        self.mask = None\n",
    "\n",
    "    def forward(self, x, train_flg=True):\n",
    "        if train_flg:\n",
    "            self.mask = np.random.rand(*x.shape) > self.dropout_ratio\n",
    "            return x * self.mask\n",
    "        else:\n",
    "            return x * (1 - self.dropout_ratio)\n",
    "\n",
    "    def backward(self, dout):\n",
    "        return dout * self.mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 超参数最优化选择\n",
    "- 各层的神经元数量，batch 大小，学习率等"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
